scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(proxy): add tpd_limit (tokens per day) for batch submissions

Browse source 
Adds a nullable tpd_limit column and field to keys, teams, budgets and end users. The batch submission limiter swaps the per-minute RPM/TPM descriptor of any scope that has a tpd_limit for a token-only 24h descriptor, so batch traffic is budgeted per day while online traffic keeps the existing per-minute limits. The Admin UI exposes the field on key, team and budget create/edit forms

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-13 10:06:07 +00:00
parent 30f33a949b
commit 438d46cb50
45 changed files with 673 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql Normal file
View file

@ -0,0 +1,11 @@
-- AlterTable
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;

2
litellm/constants.py
View file

@ -1556,6 +1556,8 @@ BASE_MCP_ROUTE: Final = "/mcp"
BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour
BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24)) # for 24 hours
BATCH_TPD_WINDOW_SECONDS: Final = 86400
BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd"
HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)) # 60 seconds
_background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS")

1
litellm/models/budget.py
View file

@ -26,6 +26,7 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
max_parallel_requests: int | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
model_max_budget: dict | None = None
budget_duration: str | None = None
allowed_models: list[str] | None = None # per-member model scope; empty = inherit team models

1
litellm/models/team.py
View file

@ -71,6 +71,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
budget_duration: str | None = None

1
litellm/models/verification_token.py
View file

@ -31,6 +31,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
metadata: dict = {}
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
allowed_cache_controls: list | None = []

7
litellm/proxy/_types.py
View file

@ -1197,6 +1197,7 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
class KeyRequestBase(GenerateRequestBase):
key: str | None = None
tpd_limit: int | None = None
default_estimated_output_tokens: PositiveInt | None = None
default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None
budget_id: str | None = None
@ -1882,6 +1883,9 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
)
tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.")
rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.")
tpd_limit: int | None = Field(
default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id."
)
budget_duration: str | None = Field(
default=None,
description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')",
@ -2052,6 +2056,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
models: list | None = None
@ -3003,6 +3008,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
team_alias: str | None = None
team_tpm_limit: int | None = None
team_rpm_limit: int | None = None
team_tpd_limit: int | None = None
team_max_budget: float | None = None
team_soft_budget: float | None = None
team_models: list = []
@ -3022,6 +3028,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
end_user_id: str | None = None
end_user_tpm_limit: int | None = None
end_user_rpm_limit: int | None = None
end_user_tpd_limit: int | None = None
end_user_max_budget: float | None = None
end_user_model_max_budget: dict | None = None

2
litellm/proxy/auth/team_grants.py
View file

@ -56,6 +56,7 @@ class TeamGrants(TypedDict, total=False):
team_alias: ReadOnly[str | None]
team_tpm_limit: ReadOnly[int | None]
team_rpm_limit: ReadOnly[int | None]
team_tpd_limit: ReadOnly[int | None]
team_max_budget: ReadOnly[float | None]
team_soft_budget: ReadOnly[float | None]
team_spend: ReadOnly[float | None]
@ -97,6 +98,7 @@ def team_grants(
team_alias=team_object.team_alias,
team_tpm_limit=team_object.tpm_limit,
team_rpm_limit=team_object.rpm_limit,
team_tpd_limit=team_object.tpd_limit,
team_max_budget=team_object.max_budget,
team_soft_budget=team_object.soft_budget,
team_spend=team_object.spend,

8
litellm/proxy/auth/user_api_key_auth.py
View file

@ -535,6 +535,9 @@ def _apply_budget_limits_to_end_user_params(
if budget_info.rpm_limit is not None:
end_user_params["end_user_rpm_limit"] = budget_info.rpm_limit
if budget_info.tpd_limit is not None:
end_user_params["end_user_tpd_limit"] = budget_info.tpd_limit
if budget_info.max_budget is not None:
end_user_params["end_user_max_budget"] = budget_info.max_budget
@ -619,6 +622,8 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use
valid_token.end_user_tpm_limit = end_user_params["end_user_tpm_limit"]
if end_user_params.get("end_user_rpm_limit") is not None:
valid_token.end_user_rpm_limit = end_user_params["end_user_rpm_limit"]
if end_user_params.get("end_user_tpd_limit") is not None:
valid_token.end_user_tpd_limit = end_user_params["end_user_tpd_limit"]
if end_user_params.get("allowed_model_region") is not None:
valid_token.allowed_model_region = end_user_params["allowed_model_region"]
if end_user_params.get("end_user_model_max_budget") is not None:
@ -2010,6 +2015,7 @@ async def _user_api_key_auth_builder(
valid_token.end_user_id = end_user_params.get("end_user_id")
valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit")
valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit")
valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit")
valid_token.allowed_model_region = end_user_params.get("allowed_model_region")
if valid_token is not None:
@ -2283,6 +2289,7 @@ async def _user_api_key_auth_builder(
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
tpd_limit=valid_token.team_tpd_limit,
blocked=valid_token.team_blocked,
models=token_team_models,
metadata=valid_token.team_metadata,
@ -2436,6 +2443,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
tpd_limit=valid_token.team_tpd_limit,
blocked=valid_token.team_blocked,
models=token_team_models,
metadata=valid_token.team_metadata,

1
litellm/proxy/db/create_views.py
View file

@ -80,6 +80,7 @@ async def create_missing_views(db: SupportsRawQueries) -> None:
t.max_budget AS team_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit,
p.project_alias AS project_alias
FROM "LiteLLM_VerificationToken" v
LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id

46
litellm/proxy/hooks/batch_rate_limiter.py
View file

@ -33,6 +33,7 @@ from litellm.batches.batch_utils import (
_extract_file_access_credentials,
_iter_batch_input_lines,
)
from litellm.constants import BATCH_TPD_DESCRIPTOR_SUFFIX, BATCH_TPD_WINDOW_SECONDS
from litellm.exceptions import RateLimitErrorCategory
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import (
@ -236,14 +237,48 @@ class _PROXY_BatchRateLimiter(CustomLogger):
file-bound/top-level routing model this function resolves. Charging
project quotas here would let a caller bind the file to a model
without a quota while rows execute against a quota-limited model.
Scopes with a ``tpd_limit`` (key, team, end user) are charged against a
daily token descriptor instead of their per-minute RPM/TPM descriptor,
because a batch's rows are scheduled by the provider and never share a
minute with the submission. The daily descriptor uses its own key so
its 24h window never collides with the online limiter's counters.
"""
return self.parallel_request_limiter._create_rate_limit_descriptors(
descriptors: Final = self.parallel_request_limiter._create_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
rpm_limit_type=None,
tpm_limit_type=None,
model_has_failures=False,
)
tpd_limits: Final[Mapping[str, tuple[str, int]]] = MappingProxyType(
{
key: (value, limit)
for key, value, limit in (
("api_key", user_api_key_dict.api_key, user_api_key_dict.tpd_limit),
("team", user_api_key_dict.team_id, user_api_key_dict.team_tpd_limit),
("end_user", user_api_key_dict.end_user_id, user_api_key_dict.end_user_tpd_limit),
)
if value and limit is not None
}
)
if not tpd_limits:
return descriptors
return [
*(d for d in descriptors if d["key"] not in tpd_limits),
*(
RateLimitDescriptor(
key=f"{key}{BATCH_TPD_DESCRIPTOR_SUFFIX}",
value=value,
rate_limit={
"requests_per_unit": None,
"tokens_per_unit": limit,
"window_size": BATCH_TPD_WINDOW_SECONDS,
},
)
for key, (value, limit) in tpd_limits.items()
),
]
@staticmethod
def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool:
@ -610,7 +645,9 @@ class _PROXY_BatchRateLimiter(CustomLogger):
)
now: Final = datetime.now().timestamp()
window_size: Final = self.parallel_request_limiter.window_size
window_size: Final = (descriptor.get("rate_limit") or {}).get(
"window_size"
) or self.parallel_request_limiter.window_size
reset_time: Final = now + window_size
reset_time_formatted: Final = datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S UTC")
@ -643,10 +680,13 @@ class _PROXY_BatchRateLimiter(CustomLogger):
if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY
else batch_usage.total_tokens
)
token_limit_label: Final = (
"TPD" if descriptor.get("key", "").endswith(BATCH_TPD_DESCRIPTOR_SUFFIX) else "TPM"
)
detail = (
f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. "
f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining "
f"out of {current_limit} TPM limit. "
f"out of {current_limit} {token_limit_label} limit. "
f"Limit resets at: {reset_time_formatted}"
)

3
litellm/proxy/management_endpoints/budget_management_endpoints.py
View file

@ -52,6 +52,7 @@ async def new_budget(
- max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
- tpm_limit: Optional[int] - The tokens per minute limit for the budget.
- rpm_limit: Optional[int] - The requests per minute limit for the budget.
- tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
- model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
- budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now.
"""
@ -135,6 +136,7 @@ async def update_budget(
- max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
- tpm_limit: Optional[int] - The tokens per minute limit for the budget.
- rpm_limit: Optional[int] - The requests per minute limit for the budget.
- tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
- model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
- budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset.
"""
@ -272,6 +274,7 @@ async def budget_settings(
"max_parallel_requests": {"type": "Integer"},
"tpm_limit": {"type": "Integer"},
"rpm_limit": {"type": "Integer"},
"tpd_limit": {"type": "Integer"},
"budget_duration": {"type": "String"},
"max_budget": {"type": "Float"},
"soft_budget": {"type": "Float"},

1
litellm/proxy/management_endpoints/customer_endpoints.py
View file

@ -335,6 +335,7 @@ async def new_end_user(
- budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
- tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute)
- rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute)
- tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit
- model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}
- max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer.
- soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests.

9
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -916,7 +916,9 @@ async def validate_team_id_used_in_service_account_request(
return True
_BUDGET_NUMERIC_KEYS = frozenset(["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit"])
_BUDGET_NUMERIC_KEYS = frozenset(
["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit", "tpd_limit"]
)
def _enforce_upperbound_key_params(
@ -1784,6 +1786,7 @@ async def generate_key_fn(
- blocked: Optional[bool] - Whether the key is blocked.
- rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
- tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
- tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
- soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
- tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
- prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
@ -1990,6 +1993,7 @@ async def generate_service_account_key_fn(
- blocked: Optional[bool] - Whether the key is blocked.
- rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
- tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
- tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
- soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
- tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
- enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
@ -2989,6 +2993,7 @@ async def update_key_fn(
- metadata: Optional[dict] - Metadata for key. Example {"team": "core-infra", "app": "app2"}
- tpm_limit: Optional[int] - Tokens per minute limit
- rpm_limit: Optional[int] - Requests per minute limit
- tpd_limit: Optional[int] - Tokens per day limit, charged by batch submissions instead of tpm_limit/rpm_limit
- model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200}
- mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200}
- tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit.
@ -4109,6 +4114,7 @@ async def generate_key_helper_fn(
metadata: dict | None = {},
tpm_limit: int | None = None,
rpm_limit: int | None = None,
tpd_limit: int | None = None,
query_type: Literal["insert_data", "update_data"] = "insert_data",
update_key_values: dict | None = None,
key_alias: str | None = None,
@ -4263,6 +4269,7 @@ async def generate_key_helper_fn(
"metadata": metadata_json,
"tpm_limit": tpm_limit,
"rpm_limit": rpm_limit,
"tpd_limit": tpd_limit,
"budget_duration": key_budget_duration,
"budget_reset_at": key_reset_at,
"allowed_cache_controls": allowed_cache_controls,

5
litellm/proxy/management_endpoints/management_v1/budgets.py
View file

@ -58,6 +58,7 @@ class BudgetListItem(BaseModel):
soft_budget: float | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
created_at: datetime
@ -123,7 +124,7 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
resource="budgets",
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at")),
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
searchable=frozenset(("budget_id",)),
filters=BUDGET_FILTERS,
default_sort=(SortKey(field="created_at", descending=True),),
@ -154,7 +155,7 @@ async def list_budgets(
way to page, sort or filter it.
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
`rpm_limit` or `created_at`, each optionally prefixed with `-` for descending,
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. `budget_id` is appended to every sort as the
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
`page_size` defaults to 50 and is capped at 100. Filters are

2
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -1215,6 +1215,7 @@ async def new_team(
- mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team.
- tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
- rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
- tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
- rpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of RPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating RPM, or "best_effort_throughput" for best effort enforcement.
- tpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of TPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating TPM, or "best_effort_throughput" for best effort enforcement.
- max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
@ -1959,6 +1960,7 @@ async def update_team(
- metadata: Optional[dict] - Metadata for team, store information for team. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
- tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
- rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
- tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
- max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
- soft_budget: Optional[float] - The soft budget threshold for the team. If max_budget is set (either in the request or existing), soft_budget must be strictly lower than max_budget. Can be set independently if max_budget is not set.
- budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets)

4
litellm/proxy/schema.prisma
View file

@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_reset_at DateTime?
@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?

5
litellm/proxy/utils.py
View file

@ -4287,7 +4287,8 @@ class PrismaClient:
t.spend AS team_spend,
t.max_budget AS team_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit
FROM "LiteLLM_VerificationToken" v
LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id;
""",
@ -4726,6 +4727,7 @@ class PrismaClient:
t.soft_budget AS team_soft_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit,
t.models AS team_models,
t.metadata AS team_metadata,
t.blocked AS team_blocked,
@ -4743,6 +4745,7 @@ class PrismaClient:
b.max_budget AS litellm_budget_table_max_budget,
b.tpm_limit AS litellm_budget_table_tpm_limit,
b.rpm_limit AS litellm_budget_table_rpm_limit,
b.tpd_limit AS litellm_budget_table_tpd_limit,
b.model_max_budget as litellm_budget_table_model_max_budget,
b.soft_budget as litellm_budget_table_soft_budget,
o.metadata as organization_metadata,

4
schema.prisma
View file

@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_reset_at DateTime?
@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?

15
tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py
View file

@ -277,3 +277,18 @@ def test_update_valid_token_db_values_override_custom_auth_when_set():
# DB values should win
assert result.end_user_tpm_limit == 500
assert result.end_user_model_max_budget == db_budget
def test_end_user_budget_tpd_limit_reaches_the_token():
from litellm.proxy.auth.user_api_key_auth import _apply_budget_limits_to_end_user_params
end_user_params = {"end_user_id": "user_1"}
_apply_budget_limits_to_end_user_params(
end_user_params=end_user_params,
budget_info=LiteLLM_BudgetTable(rpm_limit=5, tpd_limit=750000),
end_user_id="user_1",
)
result = update_valid_token_with_end_user_params(UserAPIKeyAuth(token="test_token"), end_user_params)
assert result.end_user_rpm_limit == 5
assert result.end_user_tpd_limit == 750000

2
tests/test_litellm/proxy/auth/test_team_grants.py
View file

@ -27,6 +27,7 @@ def _full_team(model_aliases=ALIASES) -> LiteLLM_TeamTable:
team_alias="grants-team",
tpm_limit=1000,
rpm_limit=10,
tpd_limit=200000,
max_budget=50.0,
soft_budget=25.0,
spend=12.5,
@ -67,6 +68,7 @@ def test_team_grants_cover_every_team_field_the_key_path_gets():
assert token.team_alias == "grants-team"
assert token.team_tpm_limit == 1000
assert token.team_rpm_limit == 10
assert token.team_tpd_limit == 200000
assert token.team_max_budget == 50.0
assert token.team_soft_budget == 25.0
assert token.team_spend == 12.5

171
tests/test_litellm/proxy/hooks/test_batch_rate_limiter.py Normal file
View file

@ -0,0 +1,171 @@
"""
Tests for `tpd_limit` (tokens per day) enforcement on batch submissions.
A batch's rows are scheduled by the provider, so a caller cannot keep a large
batch under a per-minute RPM/TPM budget. Scopes that configure `tpd_limit`
are charged against a 24h token window instead of their minute counters.
"""
import pytest
from fastapi import HTTPException
from litellm import DualCache
from litellm.constants import BATCH_TPD_WINDOW_SECONDS
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.batch_rate_limiter import BatchFileUsage
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
_PROXY_MaxParallelRequestsHandler_v3,
)
from litellm.proxy.utils import InternalUsageCache, hash_token
def _make_limiters():
internal_usage_cache = InternalUsageCache(dual_cache=DualCache())
rate_limiter = _PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=internal_usage_cache)
batch_limiter = rate_limiter._get_batch_rate_limiter()
assert batch_limiter is not None
return internal_usage_cache, rate_limiter, batch_limiter
async def _counter(internal_usage_cache, rate_limiter, descriptor_key, value, rate_limit_type):
cache_key = rate_limiter.create_rate_limit_keys(descriptor_key, value, rate_limit_type)
raw = await internal_usage_cache.async_get_cache(key=cache_key, litellm_parent_otel_span=None, local_only=True)
return int(raw or 0)
@pytest.mark.asyncio
async def test_batch_over_rpm_and_tpm_but_under_tpd_is_accepted():
internal_usage_cache, rate_limiter, batch_limiter = _make_limiters()
api_key = hash_token("tpd-key")
user_api_key_dict = UserAPIKeyAuth(api_key=api_key, rpm_limit=1, tpm_limit=10, tpd_limit=1000)
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=user_api_key_dict,
data={},
batch_usage=BatchFileUsage(total_tokens=500, request_count=50),
)
assert await _counter(internal_usage_cache, rate_limiter, "api_key_tpd", api_key, "tokens") == 500
assert await _counter(internal_usage_cache, rate_limiter, "api_key", api_key, "requests") == 0
assert await _counter(internal_usage_cache, rate_limiter, "api_key", api_key, "tokens") == 0
@pytest.mark.asyncio
async def test_cumulative_batch_tokens_over_tpd_returns_429_with_daily_reset():
_internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters()
user_api_key_dict = UserAPIKeyAuth(api_key=hash_token("tpd-key-2"), rpm_limit=1, tpd_limit=1000)
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=user_api_key_dict,
data={},
batch_usage=BatchFileUsage(total_tokens=600, request_count=6),
)
with pytest.raises(HTTPException) as exc:
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=user_api_key_dict,
data={},
batch_usage=BatchFileUsage(total_tokens=600, request_count=6),
)
assert exc.value.status_code == 429
assert "api_key_tpd" in str(exc.value.detail)
assert "600 tokens but only 400 tokens remaining out of 1000 TPD limit" in str(exc.value.detail)
assert exc.value.headers["retry-after"] == str(BATCH_TPD_WINDOW_SECONDS)
@pytest.mark.asyncio
async def test_batch_without_tpd_still_enforces_minute_rpm():
_internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters()
user_api_key_dict = UserAPIKeyAuth(api_key=hash_token("rpm-only-key"), rpm_limit=1, tpm_limit=1000)
with pytest.raises(HTTPException) as exc:
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=user_api_key_dict,
data={},
batch_usage=BatchFileUsage(total_tokens=50, request_count=5),
)
assert exc.value.status_code == 429
assert "RPM limit" in str(exc.value.detail)
@pytest.mark.asyncio
async def test_team_tpd_replaces_team_minute_limits_but_key_minute_limits_still_apply():
internal_usage_cache, rate_limiter, batch_limiter = _make_limiters()
team_key = UserAPIKeyAuth(
api_key=hash_token("team-key"),
team_id="team-1",
team_rpm_limit=1,
team_tpm_limit=10,
team_tpd_limit=5000,
)
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=team_key,
data={},
batch_usage=BatchFileUsage(total_tokens=800, request_count=8),
)
assert await _counter(internal_usage_cache, rate_limiter, "team_tpd", "team-1", "tokens") == 800
assert await _counter(internal_usage_cache, rate_limiter, "team", "team-1", "requests") == 0
key_rpm_in_team_with_tpd = UserAPIKeyAuth(
api_key=hash_token("team-key-2"),
rpm_limit=1,
team_id="team-1",
team_tpd_limit=5000,
)
with pytest.raises(HTTPException) as exc:
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=key_rpm_in_team_with_tpd,
data={},
batch_usage=BatchFileUsage(total_tokens=10, request_count=2),
)
assert exc.value.status_code == 429
assert "api_key:" in str(exc.value.detail)
assert "RPM limit" in str(exc.value.detail)
@pytest.mark.asyncio
async def test_end_user_tpd_is_enforced_per_end_user():
_internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters()
first_customer = UserAPIKeyAuth(
api_key=hash_token("shared-key"), end_user_id="customer-a", end_user_rpm_limit=1, end_user_tpd_limit=100
)
second_customer = UserAPIKeyAuth(
api_key=hash_token("shared-key"), end_user_id="customer-b", end_user_rpm_limit=1, end_user_tpd_limit=100
)
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=first_customer, data={}, batch_usage=BatchFileUsage(total_tokens=90, request_count=9)
)
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=second_customer, data={}, batch_usage=BatchFileUsage(total_tokens=90, request_count=9)
)
with pytest.raises(HTTPException) as exc:
await batch_limiter._check_and_increment_batch_counters(
user_api_key_dict=first_customer, data={}, batch_usage=BatchFileUsage(total_tokens=20, request_count=2)
)
assert exc.value.status_code == 429
assert "end_user_tpd: customer-a" in str(exc.value.detail)
def test_tpd_only_key_is_not_skipped_as_having_no_limits():
_internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters()
descriptors = batch_limiter._create_batch_rate_limit_descriptors(
user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("tpd-only"), tpd_limit=100),
data={},
)
assert batch_limiter._has_applicable_batch_rate_limits(descriptors) is True
def test_online_descriptors_ignore_tpd_limit():
_internal_usage_cache, rate_limiter, _batch_limiter = _make_limiters()
api_key = hash_token("online-key")
descriptors = rate_limiter._create_rate_limit_descriptors(
user_api_key_dict=UserAPIKeyAuth(api_key=api_key, rpm_limit=5, tpd_limit=100, team_id="t", team_tpd_limit=9),
data={"model": "gpt-4o"},
rpm_limit_type=None,
tpm_limit_type=None,
model_has_failures=False,
)
assert [(d["key"], d["rate_limit"]["window_size"]) for d in descriptors] == [("api_key", rate_limiter.window_size)]

7
tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py
View file

@ -52,7 +52,7 @@ app.include_router(router)
client = TestClient(app)
BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets"
SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpm_limit"]
SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"]
def _row(budget_id: str, **overrides: Any) -> dict[str, Any]:
@ -62,6 +62,7 @@ def _row(budget_id: str, **overrides: Any) -> dict[str, Any]:
"soft_budget": None,
"tpm_limit": None,
"rpm_limit": None,
"tpd_limit": None,
"budget_duration": "30d",
"budget_reset_at": None,
"created_at": "2026-07-20T12:00:00+00:00",
@ -123,7 +124,7 @@ def test_returns_flat_rows_in_the_control_plane_envelope(query_raw, as_proxy_adm
def test_serves_the_columns_the_budgets_page_renders(query_raw, as_proxy_admin):
_serve(query_raw, [_row("b-1", soft_budget=5.0, budget_reset_at="2026-08-01T00:00:00+00:00")])
_serve(query_raw, [_row("b-1", soft_budget=5.0, tpd_limit=250000, budget_reset_at="2026-08-01T00:00:00+00:00")])
row = _get().json()["data"][0]
@ -133,12 +134,14 @@ def test_serves_the_columns_the_budgets_page_renders(query_raw, as_proxy_admin):
"soft_budget",
"tpm_limit",
"rpm_limit",
"tpd_limit",
"budget_duration",
"budget_reset_at",
"created_at",
"updated_at",
}
assert row["soft_budget"] == 5.0
assert row["tpd_limit"] == 250000
assert row["budget_reset_at"].startswith("2026-08-01T00:00:00")

15
tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py
View file

@ -136,6 +136,21 @@ async def test_update_budget_success(client_and_mocks, monkeypatch):
assert body["updated_by"] == "test_user"
@pytest.mark.asyncio
async def test_new_and_update_budget_persist_tpd_limit(client_and_mocks):
client, _, mock_table = client_and_mocks
resp = client.post("/budget/new", json={"budget_id": "budget_tpd", "tpd_limit": 250000})
assert resp.status_code == 200, resp.text
assert resp.json()["tpd_limit"] == 250000
assert mock_table.create.await_args.kwargs["data"]["tpd_limit"] == 250000
resp = client.post("/budget/update", json={"budget_id": "budget_tpd", "tpd_limit": 500000})
assert resp.status_code == 200, resp.text
assert resp.json()["tpd_limit"] == 500000
assert mock_table.update.await_args.kwargs["data"]["tpd_limit"] == 500000
@pytest.mark.asyncio
async def test_update_budget_missing_id(client_and_mocks, monkeypatch):
client, mock_prisma, mock_table = client_and_mocks

34
tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
View file

@ -461,6 +461,28 @@ async def test_key_expiration_exact_duration_hours(monkeypatch):
), f"Expected expiration to be approximately 12 hours from creation, got {hours_diff} hours"
@pytest.mark.asyncio
async def test_generate_key_persists_tpd_limit(monkeypatch):
mock_prisma_client = AsyncMock()
mock_prisma_client.insert_data = AsyncMock(
return_value=MagicMock(token="hashed_token_123", litellm_budget_table=None)
)
mock_prisma_client.db = MagicMock()
mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
data_json = GenerateKeyRequest(tpd_limit=250000, rpm_limit=5).model_dump(exclude_none=True)
response = await generate_key_helper_fn(request_type="key", **data_json, table_name="key")
assert response["tpd_limit"] == 250000
key_insert = mock_prisma_client.insert_data.await_args_list[-1].kwargs
assert key_insert["table_name"] == "key"
assert key_insert["data"]["tpd_limit"] == 250000
assert key_insert["data"]["rpm_limit"] == 5
@pytest.mark.asyncio
async def test_key_generation_with_object_permission(monkeypatch):
"""Ensure /key/generate correctly handles `object_permission` input by
@ -1813,6 +1835,18 @@ async def test_update_key_enable_prompt_caching_folds_into_metadata(flag_value):
assert "enable_prompt_caching" not in {k for k in updated if k != "metadata"}
@pytest.mark.asyncio
@pytest.mark.parametrize("tpd_limit", [250000, None])
async def test_update_key_writes_tpd_limit_as_a_column(tpd_limit):
data = UpdateKeyRequest(key="sk-1", tpd_limit=tpd_limit)
existing_key = LiteLLM_VerificationToken(token="hashed", tpd_limit=1)
updated = await prepare_key_update_data(data=data, existing_key_row=existing_key)
assert updated["tpd_limit"] == tpd_limit
assert "rpm_limit" not in updated
@pytest.mark.asyncio
async def test_update_preserves_service_account_id_when_metadata_replaced():
"""

78
tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
View file

@ -626,6 +626,42 @@ async def test_new_team_with_object_permission(mock_db_client, mock_admin_auth):
assert "object_permission" not in team_data
@pytest.mark.asyncio
async def test_new_team_persists_tpd_limit(mock_db_client, mock_admin_auth):
mock_db_client.jsonify_team_object = lambda db_data: db_data
mock_db_client.get_data = AsyncMock(return_value=None)
mock_db_client.update_data = AsyncMock(return_value=MagicMock())
mock_db_client.db = MagicMock()
mock_db_client.db.litellm_modeltable = MagicMock()
mock_db_client.db.litellm_modeltable.create = AsyncMock(return_value=MagicMock(id="model123"))
team_create_result = MagicMock(team_id="team-tpd")
team_create_result.model_dump.return_value = {"team_id": "team-tpd", "tpd_limit": 250000}
mock_team_create = AsyncMock(return_value=team_create_result)
mock_db_client.db.litellm_teamtable = MagicMock()
mock_db_client.db.litellm_teamtable.create = mock_team_create
_wire_team_create_tx(mock_db_client)
mock_db_client.db.litellm_teamtable.count = AsyncMock(return_value=0)
mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=team_create_result)
mock_db_client.db.litellm_usertable = MagicMock()
mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock())
from fastapi import Request
from litellm.proxy._types import NewTeamRequest
from litellm.proxy.management_endpoints.team_endpoints import new_team
await new_team(
data=NewTeamRequest(team_alias="tpd-team", rpm_limit=5, tpd_limit=250000),
http_request=MagicMock(spec=Request),
user_api_key_dict=mock_admin_auth,
)
team_data = mock_team_create.call_args.kwargs["data"]
assert team_data["tpd_limit"] == 250000
assert team_data["rpm_limit"] == 5
@pytest.mark.asyncio
async def test_new_team_with_mcp_tool_permissions(mock_db_client, mock_admin_auth):
"""
@ -7338,6 +7374,48 @@ async def test_update_team_rpm_limit_not_gated_by_user_limit(
assert result is not None
@pytest.mark.asyncio
async def test_update_team_persists_tpd_limit(disable_audit_logging_for_mocked_team):
from fastapi import Request
from litellm.proxy._types import UpdateTeamRequest, UserAPIKeyAuth
from litellm.proxy.management_endpoints.team_endpoints import update_team
with (
patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point
"litellm.proxy.proxy_server.prisma_client"
) as mock_prisma,
patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point
"litellm.proxy.proxy_server.user_api_key_cache"
) as mock_cache,
patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point
"litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"
),
patch( # test-quality-ok: stubs the audit write so the test observes only the team column written
"litellm.proxy.proxy_server.create_audit_log_for_update", new=AsyncMock()
),
):
existing_team = MagicMock(team_id="team-tpd", organization_id=None, model_id=None, tpd_limit=None)
existing_team.model_dump.return_value = {"team_id": "team-tpd", "organization_id": None}
mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing_team)
mock_prisma.jsonify_team_object = lambda db_data: db_data
mock_cache.async_get_cache = AsyncMock(return_value=None)
mock_cache.async_set_cache = AsyncMock()
updated_team = MagicMock(team_id="team-tpd", organization_id=None, litellm_model_table=None)
updated_team.model_dump.return_value = {"team_id": "team-tpd", "organization_id": None, "tpd_limit": 250000}
mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=updated_team)
await update_team(
data=UpdateTeamRequest(team_id="team-tpd", tpd_limit=250000),
http_request=MagicMock(spec=Request),
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"),
)
written = mock_prisma.db.litellm_teamtable.update.call_args.kwargs["data"]
assert written["tpd_limit"] == 250000
assert "rpm_limit" not in written
@pytest.mark.asyncio
async def test_new_team_org_scoped_tpm_exceeds_org_limit():
"""

8
ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx
View file

@ -129,7 +129,7 @@ describe("BudgetTable", () => {
const user = userEvent.setup();
renderWithProviders(<BudgetTable {...defaultProps} list={makeList()} />);
await showColumn(user, "created_at");
for (const field of ["budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at"]) {
for (const field of ["budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at"]) {
expect(screen.getByTestId(`sort-header-${field}`)).toBeInTheDocument();
}
});
@ -152,9 +152,11 @@ describe("BudgetTable", () => {
});
it("should show n/a for missing rate limits and Unlimited for a missing max budget", () => {
const list = makeList({ rows: [makeBudget({ max_budget: null, tpm_limit: null, rpm_limit: null })] });
const list = makeList({
rows: [makeBudget({ max_budget: null, tpm_limit: null, rpm_limit: null, tpd_limit: null })],
});
renderWithProviders(<BudgetTable {...defaultProps} list={list} />);
expect(screen.getAllByText("n/a")).toHaveLength(2);
expect(screen.getAllByText("n/a")).toHaveLength(3);
expect(screen.getByText("Unlimited")).toBeInTheDocument();
});

8
ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx
View file

@ -126,6 +126,14 @@ export const getBudgetTableColumns = ({
size: 100,
cell: ({ row }) => <RateLimitCell value={row.original.rpm_limit} />,
},
{
id: "tpd_limit",
accessorKey: "tpd_limit",
meta: { title: "TPD (batch)", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="TPD (batch)" />,
size: 110,
cell: ({ row }) => <RateLimitCell value={row.original.tpd_limit} />,
},
{
id: "budget_duration",
accessorKey: "budget_duration",

18
ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx
View file

@ -17,6 +17,7 @@ const budgetShape = {
budget_id: z.string().min(1, "Please input a human-friendly name for the budget"),
tpm_limit: z.number().nullish(),
rpm_limit: z.number().nullish(),
tpd_limit: z.number().nullish(),
max_budget: z.number().nullish(),
budget_duration: z.string().nullish(),
};
@ -112,6 +113,23 @@ const BudgetModal: React.FC<BudgetModalProps> = ({ isModalVisible, setIsModalVis
/>
)}
</FormField>
<FormField
control={form.control}
name="tpd_limit"
label="Max Tokens per day (batch)"
description="Daily token budget for batch submissions. When set, batches are charged against this instead of TPM/RPM."
>
{({ ref, value, onChange, ...field }) => (
<Input
{...field}
ref={ref}
type="number"
step={1}
value={value ?? ""}
onChange={(event) => onChange(event.target.value === "" ? null : event.target.valueAsNumber)}
/>
)}
</FormField>
<Collapsible open={optionalSettingsOpen} onOpenChange={setOptionalSettingsOpen} className="mt-20 mb-8">
<CollapsibleTrigger className="group flex w-full items-center justify-between py-2 text-left">

1
ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx
View file

@ -133,6 +133,7 @@ const BudgetPanel: React.FC<BudgetSettingsPageProps> = ({ accessToken }) => {
{ label: "Max Budget", value: selectedBudget?.max_budget },
{ label: "TPM", value: selectedBudget?.tpm_limit },
{ label: "RPM", value: selectedBudget?.rpm_limit },
{ label: "TPD (batch)", value: selectedBudget?.tpd_limit },
]}
onCancel={handleDeleteCancel}
onOk={handleDeleteConfirm}

20
ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx
View file

@ -15,13 +15,14 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u
type EditBudgetFormValues = Pick<
budgetItem,
"budget_id" | "tpm_limit" | "rpm_limit" | "max_budget" | "budget_duration"
"budget_id" | "tpm_limit" | "rpm_limit" | "tpd_limit" | "max_budget" | "budget_duration"
>;
const toFormValues = (budget: budgetItem): EditBudgetFormValues => ({
budget_id: budget.budget_id,
tpm_limit: budget.tpm_limit,
rpm_limit: budget.rpm_limit,
tpd_limit: budget.tpd_limit,
max_budget: budget.max_budget,
budget_duration: budget.budget_duration,
});
@ -118,6 +119,23 @@ const EditBudgetModal: React.FC<EditBudgetModalProps> = ({ isModalVisible, setIs
/>
)}
</FormField>
<FormField
control={form.control}
name="tpd_limit"
label="Max Tokens per day (batch)"
description="Daily token budget for batch submissions. When set, batches are charged against this instead of TPM/RPM."
>
{({ ref, value, onChange, ...field }) => (
<Input
{...field}
ref={ref}
type="number"
step={1}
value={value ?? ""}
onChange={(event) => onChange(event.target.value === "" ? null : event.target.valueAsNumber)}
/>
)}
</FormField>
<Collapsible open={optionalSettingsOpen} onOpenChange={setOptionalSettingsOpen} className="mt-20 mb-8">
<CollapsibleTrigger className="group flex w-full items-center justify-between py-2 text-left">

4
ui/litellm-dashboard/src/components/Teams.test.tsx
View file

@ -1187,6 +1187,7 @@ describe("Teams - which fields reach the create payload depends on the open sect
"organization_id",
"rpm_limit",
"team_alias",
"tpd_limit",
"tpm_limit",
]);
expect(payload.team_alias).toBe("Closed Sections Team");
@ -1314,6 +1315,7 @@ describe("Teams - the exact bytes the create call sends", () => {
budget_duration: undefined,
tpm_limit: undefined,
rpm_limit: undefined,
tpd_limit: undefined,
metadata: undefined,
});
expect(wireBody(payload)).toStrictEqual({
@ -1341,6 +1343,7 @@ describe("Teams - the exact bytes the create call sends", () => {
budget_duration: undefined,
tpm_limit: undefined,
rpm_limit: undefined,
tpd_limit: undefined,
metadata: undefined,
team_id: undefined,
team_member_budget: undefined,
@ -1513,6 +1516,7 @@ describe("Teams - the exact bytes the create call sends", () => {
budget_duration: undefined,
tpm_limit: undefined,
rpm_limit: undefined,
tpd_limit: undefined,
metadata: undefined,
team_id: undefined,
team_member_budget: undefined,

14
ui/litellm-dashboard/src/components/Teams.tsx
View file

@ -77,6 +77,7 @@ const teamCreateFieldsSchema = z.object({
budget_duration: z.string().nullish(),
tpm_limit: numericInputSchema,
rpm_limit: numericInputSchema,
tpd_limit: numericInputSchema,
metadata: metadataPairsSchema.optional(),
team_id: z.string().optional(),
team_member_budget: z.number().optional(),
@ -113,6 +114,7 @@ const EMPTY_TEAM_CREATE_VALUES: TeamCreateFormValues = {
budget_duration: undefined,
tpm_limit: undefined,
rpm_limit: undefined,
tpd_limit: undefined,
metadata: [],
team_id: undefined,
team_member_budget: undefined,
@ -821,6 +823,18 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, premiumUser
<NumericalInput {...field} ref={ref} value={value ?? ""} step={1} width={400} />
)}
</FormField>
<FormField
control={form.control}
name="tpd_limit"
label={labelWithHint(
"Tokens per day Limit (TPD)",
"Daily token budget for batch submissions (/v1/batches). When set, batch input files are charged against this 24h window instead of the team's TPM/RPM limits. Online requests keep using TPM/RPM.",
)}
>
{({ ref, value, ...field }) => (
<NumericalInput {...field} ref={ref} value={value ?? ""} step={1} width={400} />
)}
</FormField>
<Field>
<FieldLabel>Metadata</FieldLabel>
<MetadataKeyValueFields

2
ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx
View file

@ -12,6 +12,7 @@ export interface Team {
budget_duration: string | null;
tpm_limit: number | null;
rpm_limit: number | null;
tpd_limit?: number | null;
organization_id: string;
metadata?: Record<string, unknown> | null;
budget_reset_at?: string | null;
@ -47,6 +48,7 @@ export interface KeyResponse {
metadata: Record<string, unknown>;
tpm_limit: number;
rpm_limit: number;
tpd_limit?: number | null;
duration: string;
budget_duration: string;
budget_reset_at: string;

18
ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts
View file

@ -45,6 +45,7 @@ const DROPPED_AT_SERIALISATION = [
"rpm_limit",
"tags",
"throttle_on_budget_exceeded",
"tpd_limit",
"tpm_limit",
];
@ -64,6 +65,7 @@ const OPTIONAL_SETTINGS_VALUES = {
tpm_limit_type: "key",
rpm_limit: undefined,
rpm_limit_type: "key",
tpd_limit: undefined,
throttle_on_budget_exceeded: undefined,
enable_prompt_caching: undefined,
guardrails: undefined,
@ -456,6 +458,18 @@ describe("budget duration", () => {
});
});
describe("tpd_limit", () => {
it("forwards the daily batch token budget alongside the minute limits", () => {
expect(payloadOf(build({ key_alias: "my-key", tpd_limit: 250000, rpm_limit: 5 }))).toStrictEqual(
aliasOnly({ tpd_limit: 250000, rpm_limit: 5 }),
);
});
it("keeps a zero tpd_limit rather than treating it as unset", () => {
expect(payloadOf(build({ key_alias: "my-key", tpd_limit: 0 }))).toStrictEqual(aliasOnly({ tpd_limit: 0 }));
});
});
describe("purity", () => {
it("leaves the submitted form values untouched", () => {
const values = {
@ -499,9 +513,9 @@ describe("serialised wire shape", () => {
expect(payloadOf(build({ ...CLOSED_SECTIONS_VALUES, team_id: "team-1" })).team_id).toBe("team-1");
});
it("adds fifteen keys to the object and only the two limit types to the wire when Optional Settings opens", () => {
it("adds sixteen keys to the object and only the two limit types to the wire when Optional Settings opens", () => {
const payload = payloadOf(build(OPTIONAL_SETTINGS_VALUES));
expect(Object.keys(payload)).toHaveLength(23);
expect(Object.keys(payload)).toHaveLength(24);
expect(wireKeys(payload)).toStrictEqual([
"team_id",
"key_alias",

2
ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx
View file

@ -143,6 +143,7 @@ const OPTIONAL_OPEN_PAYLOAD = {
tpm_limit_type: null,
rpm_limit: undefined,
rpm_limit_type: null,
tpd_limit: undefined,
throttle_on_budget_exceeded: undefined,
enable_prompt_caching: undefined,
guardrails: undefined,
@ -395,6 +396,7 @@ describe("CreateKey", () => {
it.each([
["Tokens per minute Limit (TPM)", "tpm_limit"],
["Requests per minute Limit (RPM)", "rpm_limit"],
["Tokens per day Limit (TPD)", "tpd_limit"],
])("routes a typed %s into the %s payload key", async (label, key) => {
await openModal();
await nameTheKey();

27
ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
View file

@ -1150,6 +1150,32 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
/>
)}
</MountedFormField>
<MountedFormField
className="mt-4"
label={
<span>
Tokens per day Limit (TPD){" "}
<SimpleTooltip content="Daily token budget for batch submissions (/v1/batches). When set, batch input files are charged against this 24h window instead of the key's TPM/RPM limits. Online requests keep using TPM/RPM.">
<Info className="ml-1 inline size-3.5 align-text-bottom" />
</SimpleTooltip>
</span>
}
name="tpd_limit"
help={`TPD cannot exceed team TPD limit: ${team?.tpd_limit !== null && team?.tpd_limit !== undefined ? team?.tpd_limit : "unlimited"}`}
rules={ceilingRule(
team?.tpd_limit,
(limit) => `TPD limit cannot exceed team TPD limit: ${limit}`,
)}
>
{(control) => (
<NumericalInput
{...control}
value={control.value as number | string | undefined}
step={1}
width={400}
/>
)}
</MountedFormField>
<Field className="mt-4">
<FieldLabel>
<span>
@ -1760,6 +1786,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
"budget_duration",
"tpm_limit",
"rpm_limit",
"tpd_limit",
...(disableCustomApiKeys ? ["key"] : []),
]}
/>

1
ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
View file

@ -1968,6 +1968,7 @@ describe("TeamInfoView - the exact bytes the update call sends", () => {
models: ["gpt-4"],
tpm_limit: 1000,
rpm_limit: 1000,
tpd_limit: null,
model_tpm_limit: {},
model_rpm_limit: {},
max_budget: 100,

18
ui/litellm-dashboard/src/components/team/TeamInfo.tsx
View file

@ -264,6 +264,7 @@ export interface TeamData {
metadata: Record<string, any>;
tpm_limit: number | null;
rpm_limit: number | null;
tpd_limit?: number | null;
max_budget: number | null;
soft_budget?: number | null;
budget_duration: string | null;
@ -330,6 +331,7 @@ const teamUpdateFieldsSchema = z.object({
budget_duration: z.string().nullish(),
tpm_limit: numericInputSchema,
rpm_limit: numericInputSchema,
tpd_limit: numericInputSchema,
modelLimits: z
.array(
z.object({
@ -411,6 +413,7 @@ const EMPTY_TEAM_UPDATE_VALUES: TeamUpdateFormValues = {
budget_duration: undefined,
tpm_limit: undefined,
rpm_limit: undefined,
tpd_limit: undefined,
modelLimits: [],
default_estimated_output_tokens: undefined,
default_estimated_output_tokens_per_model: "",
@ -460,6 +463,7 @@ const toTeamFormValues = (info: TeamInfoRecord, effectiveGuardrails: string[]):
budget_duration: info.budget_duration,
tpm_limit: info.tpm_limit,
rpm_limit: info.rpm_limit,
tpd_limit: info.tpd_limit,
modelLimits: Array.from(
new Set([
...Object.keys(info.metadata?.model_tpm_limit ?? {}),
@ -918,6 +922,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
models: normalizeTeamModelSelection(values.models),
tpm_limit: sanitizeNumeric(values.tpm_limit),
rpm_limit: sanitizeNumeric(values.rpm_limit),
tpd_limit: sanitizeNumeric(values.tpd_limit),
model_tpm_limit: modelTpmLimit,
model_rpm_limit: modelRpmLimit,
max_budget: values.max_budget,
@ -1168,6 +1173,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
<div className="mt-2">
<p>TPM: {info.tpm_limit ?? "Unlimited"}</p>
<p>RPM: {info.rpm_limit ?? "Unlimited"}</p>
<p>TPD (batch): {info.tpd_limit ?? "Unlimited"}</p>
{info.max_parallel_requests && <p>Max Parallel Requests: {info.max_parallel_requests}</p>}
{(() => {
const modelTpm = (info.metadata?.model_tpm_limit ?? {}) as Record<string, number>;
@ -1538,6 +1544,17 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
{({ ref, value, ...field }) => <NumericalInput {...field} ref={ref} value={value ?? ""} step={1} />}
</FormField>
<FormField
control={form.control}
name="tpd_limit"
label={labelWithHint(
"Tokens per day Limit (TPD)",
"Daily token budget for batch submissions (/v1/batches). When set, batch input files are charged against this 24h window instead of the team's TPM/RPM limits. Online requests keep using TPM/RPM.",
)}
>
{({ ref, value, ...field }) => <NumericalInput {...field} ref={ref} value={value ?? ""} step={1} />}
</FormField>
<Field>
<FieldLabel>Metadata</FieldLabel>
<MetadataKeyValueFields
@ -1997,6 +2014,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
<p className="font-medium">Rate Limits</p>
<div>TPM: {info.tpm_limit ?? "Unlimited"}</div>
<div>RPM: {info.rpm_limit ?? "Unlimited"}</div>
<div>TPD (batch): {info.tpd_limit ?? "Unlimited"}</div>
{(() => {
const modelTpm = (info.metadata?.model_tpm_limit ?? {}) as Record<string, number>;
const modelRpm = (info.metadata?.model_rpm_limit ?? {}) as Record<string, number>;

3
ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx
View file

@ -58,6 +58,9 @@ export const KeyTypeSelect = ({
const SKILLS_HINT =
"Enabled skills are visible to every key. Grant disabled (private) Claude Code plugins to this key here.";
export const TPD_HINT =
"Daily token budget for batch submissions (/v1/batches). When set, batch input files are charged against this 24h window instead of the key's TPM/RPM limits. Online requests keep using TPM/RPM.";
export const KeyAgentAndSkillFields = ({
control,
accessToken,

24
ui/litellm-dashboard/src/components/templates/keyEditFormValues.test.ts
View file

@ -1,8 +1,30 @@
import { describe, expect, it } from "vitest";
import { keyEditFormSchema } from "./keyEditFormValues";
import type { KeyResponse } from "../key_team_helpers/key_list";
import { keyEditFormSchema, toKeyEditFormValues, toSubmittedValues } from "./keyEditFormValues";
const parse = (values: Record<string, unknown>) => keyEditFormSchema.safeParse(values);
describe("tpd_limit round trip", () => {
const keyData = { token: "tok", models: [], rpm_limit: 5, tpd_limit: 250000 } as unknown as KeyResponse;
it("hydrates the stored daily batch budget into the edit form", () => {
expect(toKeyEditFormValues(keyData)).toMatchObject({ rpm_limit: 5, tpd_limit: 250000 });
});
it("submits tpd_limit next to the minute limits", () => {
const submitted = toSubmittedValues(toKeyEditFormValues(keyData), { canViewPolicies: true, canViewPrompts: true });
expect(submitted).toMatchObject({ rpm_limit: 5, tpd_limit: 250000 });
});
it("submits null when the operator cleared tpd_limit", () => {
const submitted = toSubmittedValues(
{ ...toKeyEditFormValues(keyData), tpd_limit: null },
{ canViewPolicies: true, canViewPrompts: true },
);
expect(submitted.tpd_limit).toBeNull();
});
});
describe("keyEditFormSchema", () => {
it("accepts an empty form", () => {
expect(parse({}).success).toBe(true);

4
ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts
View file

@ -28,6 +28,7 @@ export interface KeyEditFormValues {
tpm_limit_type?: string | null;
rpm_limit?: number | string | null;
rpm_limit_type?: string | null;
tpd_limit?: number | string | null;
throttle_on_budget_exceeded?: boolean;
enable_prompt_caching?: boolean;
max_parallel_requests?: number | string | null;
@ -77,6 +78,7 @@ export const toKeyEditFormValues = (keyData: KeyResponse): KeyEditFormValues =>
tpm_limit_type: (keyData as { tpm_limit_type?: string | null }).tpm_limit_type ?? null,
rpm_limit: keyData.rpm_limit,
rpm_limit_type: (keyData as { rpm_limit_type?: string | null }).rpm_limit_type ?? null,
tpd_limit: keyData.tpd_limit,
throttle_on_budget_exceeded: Boolean(readMetadata(keyData, "throttle_on_budget_exceeded")),
enable_prompt_caching: Boolean(readMetadata(keyData, "enable_prompt_caching")),
max_parallel_requests: keyData.max_parallel_requests,
@ -130,6 +132,7 @@ export const keyEditFormSchema = z.object({
tpm_limit_type: z.custom<string | null | undefined>(),
rpm_limit: z.custom<number | string | null | undefined>(),
rpm_limit_type: z.custom<string | null | undefined>(),
tpd_limit: z.custom<number | string | null | undefined>(),
throttle_on_budget_exceeded: z.custom<boolean | undefined>(),
enable_prompt_caching: z.custom<boolean | undefined>(),
max_parallel_requests: z.custom<number | string | null | undefined>(),
@ -184,6 +187,7 @@ export const toSubmittedValues = (
tpm_limit_type: values.tpm_limit_type,
rpm_limit: values.rpm_limit,
rpm_limit_type: values.rpm_limit_type,
tpd_limit: values.tpd_limit,
throttle_on_budget_exceeded: values.throttle_on_budget_exceeded,
enable_prompt_caching: values.enable_prompt_caching,
max_parallel_requests: values.max_parallel_requests,

2
ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx
View file

@ -188,6 +188,7 @@ describe("KeyEditView", () => {
},
tpm_limit: 10,
rpm_limit: 10,
tpd_limit: 250000,
duration: "30d",
budget_duration: "30d",
budget_reset_at: "never",
@ -1986,6 +1987,7 @@ describe("KeyEditView", () => {
tpm_limit_type: null,
rpm_limit: 10,
rpm_limit_type: null,
tpd_limit: 250000,
throttle_on_budget_exceeded: false,
enable_prompt_caching: false,
max_parallel_requests: 10,

12
ui/litellm-dashboard/src/components/templates/key_edit_view.tsx
View file

@ -31,7 +31,13 @@ import {
modelSentinelOptions,
parseAllowedRoutes,
} from "./keyEditFieldNormalizers";
import { KeyAgentAndSkillFields, KeyBudgetNumberField, KeyTypeSelect, labelWithHint } from "./KeyEditViewControls";
import {
KeyAgentAndSkillFields,
KeyBudgetNumberField,
KeyTypeSelect,
labelWithHint,
TPD_HINT,
} from "./KeyEditViewControls";
import {
KeyEditFormValues,
keyEditFormSchema,
@ -508,6 +514,10 @@ export function KeyEditView({
)}
</FormField>
<FormField control={form.control} name="tpd_limit" label={labelWithHint("TPD Limit (batch)", TPD_HINT)}>
{({ ref: _ref, ...field }) => <NumericalInput {...field} value={field.value ?? ""} min={0} />}
</FormField>
<FormField
control={form.control}
name="throttle_on_budget_exceeded"

3
ui/litellm-dashboard/src/components/templates/key_info_view.tsx
View file

@ -288,6 +288,7 @@ export default function KeyInfoView({
formValues.max_budget = mapEmptyStringToNull(formValues.max_budget);
formValues.tpm_limit = mapEmptyStringToNull(formValues.tpm_limit);
formValues.rpm_limit = mapEmptyStringToNull(formValues.rpm_limit);
formValues.tpd_limit = mapEmptyStringToNull(formValues.tpd_limit);
formValues.max_parallel_requests = mapEmptyStringToNull(formValues.max_parallel_requests);
// Convert metadata back to an object if it exists and is a string
@ -688,6 +689,7 @@ export default function KeyInfoView({
<p className="text-sm">
RPM: {currentKeyData.rpm_limit !== null ? currentKeyData.rpm_limit : "Unlimited"}
</p>
<p className="text-sm">TPD (batch): {currentKeyData.tpd_limit ?? "Unlimited"}</p>
{Boolean(currentKeyData.metadata?.throttle_on_budget_exceeded) && (
<p className="text-sm">Throttle on budget exceeded: Yes</p>
)}
@ -1064,6 +1066,7 @@ export default function KeyInfoView({
<p className="text-sm">
RPM: {currentKeyData.rpm_limit !== null ? currentKeyData.rpm_limit : "Unlimited"}
</p>
<p className="text-sm">TPD (batch): {currentKeyData.tpd_limit ?? "Unlimited"}</p>
<p className="text-sm">
Max Parallel Requests:{" "}
{currentKeyData.max_parallel_requests !== null

69
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -1843,6 +1843,7 @@ export interface paths {
* - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
* - tpm_limit: Optional[int] - The tokens per minute limit for the budget.
* - rpm_limit: Optional[int] - The requests per minute limit for the budget.
* - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
* - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
* - budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now.
*/
@ -1899,6 +1900,7 @@ export interface paths {
* - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
* - tpm_limit: Optional[int] - The tokens per minute limit for the budget.
* - rpm_limit: Optional[int] - The requests per minute limit for the budget.
* - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
* - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
* - budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset.
*/
@ -3951,6 +3953,7 @@ export interface paths {
* - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
* - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute)
* - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute)
* - tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit
* - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}
* - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer.
* - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests.
@ -4485,6 +4488,7 @@ export interface paths {
* - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
* - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute)
* - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute)
* - tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit
* - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}
* - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer.
* - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests.
@ -7707,6 +7711,7 @@ export interface paths {
* - blocked: Optional[bool] - Whether the key is blocked.
* - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
* - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
* - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
* - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
* - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
* - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
@ -8020,6 +8025,7 @@ export interface paths {
* - blocked: Optional[bool] - Whether the key is blocked.
* - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
* - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
* - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
* - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
* - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
* - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
@ -8146,6 +8152,7 @@ export interface paths {
* - metadata: Optional[dict] - Metadata for key. Example {"team": "core-infra", "app": "app2"}
* - tpm_limit: Optional[int] - Tokens per minute limit
* - rpm_limit: Optional[int] - Requests per minute limit
* - tpd_limit: Optional[int] - Tokens per day limit, charged by batch submissions instead of tpm_limit/rpm_limit
* - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200}
* - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200}
* - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit.
@ -8395,7 +8402,7 @@ export interface paths {
* way to page, sort or filter it.
*
* `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
* `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending,
* `rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
* and defaults to `-created_at`. `budget_id` is appended to every sort as the
* tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
* `page_size` defaults to 50 and is capped at 100. Filters are
@ -15463,6 +15470,7 @@ export interface paths {
* - mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team.
* - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
* - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
* - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
* - rpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of RPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating RPM, or "best_effort_throughput" for best effort enforcement.
* - tpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of TPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating TPM, or "best_effort_throughput" for best effort enforcement.
* - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
@ -15691,6 +15699,7 @@ export interface paths {
* - metadata: Optional[dict] - Metadata for team, store information for team. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
* - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
* - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
* - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
* - soft_budget: Optional[float] - The soft budget threshold for the team. If max_budget is set (either in the request or existing), soft_budget must be strictly lower than max_budget. Can be set independently if max_budget is not set.
* - budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets)
@ -16781,7 +16790,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -16887,7 +16895,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -24442,6 +24449,8 @@ export interface components {
rpm_limit?: number | null;
/** Soft Budget */
soft_budget?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/**
@ -24494,6 +24503,11 @@ export interface components {
* @description Requests will NOT fail if this is exceeded. Will fire alerting though.
*/
soft_budget?: number | null;
/**
* Tpd Limit
* @description Max tokens per day, charged by batch submissions, allowed for this budget id.
*/
tpd_limit?: number | null;
/**
* Tpm Limit
* @description Max tokens per minute, allowed for this budget id.
@ -27856,6 +27870,8 @@ export interface components {
team_id?: string | null;
/** Throttle On Budget Exceeded */
throttle_on_budget_exceeded?: boolean | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Type */
@ -28016,6 +28032,8 @@ export interface components {
token?: string | null;
/** Token Id */
token_id?: string | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Type */
@ -28746,6 +28764,8 @@ export interface components {
rpm_limit?: number | null;
/** Soft Budget */
soft_budget?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -28779,6 +28799,8 @@ export interface components {
rpm_limit?: number | null;
/** Soft Budget */
soft_budget?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -28887,6 +28909,8 @@ export interface components {
team_id: string;
/** Team Member Permissions */
team_member_permissions?: string[] | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Updated At */
@ -29054,6 +29078,8 @@ export interface components {
team_id?: string | null;
/** Token */
token?: string | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Updated At */
@ -30241,6 +30267,8 @@ export interface components {
team_id: string;
/** Team Member Permissions */
team_member_permissions?: string[] | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Updated At */
@ -30617,6 +30645,8 @@ export interface components {
team_id?: string | null;
/** Token */
token?: string | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Updated At */
@ -32260,6 +32290,11 @@ export interface components {
soft_budget?: number | null;
/** Spend */
spend?: number | null;
/**
* Tpd Limit
* @description Max tokens per day, charged by batch submissions, allowed for this budget id.
*/
tpd_limit?: number | null;
/**
* Tpm Limit
* @description Max tokens per minute, allowed for this budget id.
@ -32475,6 +32510,8 @@ export interface components {
rpm_limit?: number | null;
/** Soft Budget */
soft_budget?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -32596,6 +32633,8 @@ export interface components {
tags?: string[] | null;
/** Team Id */
team_id: string;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -32782,6 +32821,8 @@ export interface components {
team_member_rpm_limit?: number | null;
/** Team Member Tpm Limit */
team_member_tpm_limit?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Type */
@ -33075,6 +33116,8 @@ export interface components {
token?: string | null;
/** Token Id */
token_id?: string | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Type */
@ -33531,6 +33574,8 @@ export interface components {
team_member_rpm_limit?: number | null;
/** Team Member Tpm Limit */
team_member_tpm_limit?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -34940,6 +34985,8 @@ export interface components {
team_id?: string | null;
/** Throttle On Budget Exceeded */
throttle_on_budget_exceeded?: boolean | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Type */
@ -36917,6 +36964,8 @@ export interface components {
team_id: string;
/** Team Member Permissions */
team_member_permissions?: string[] | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Updated At */
@ -37057,6 +37106,8 @@ export interface components {
team_id: string;
/** Team Member Permissions */
team_member_permissions?: string[] | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Updated At */
@ -38124,6 +38175,8 @@ export interface components {
temp_budget_increase?: number | null;
/** Throttle On Budget Exceeded */
throttle_on_budget_exceeded?: boolean | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Type */
@ -38388,6 +38441,8 @@ export interface components {
tags?: string[] | null;
/** Team Id */
team_id?: string | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -38583,6 +38638,8 @@ export interface components {
team_member_rpm_limit?: number | null;
/** Team Member Tpm Limit */
team_member_tpm_limit?: number | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
};
@ -39093,6 +39150,8 @@ export interface components {
end_user_object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null;
/** End User Rpm Limit */
end_user_rpm_limit?: number | null;
/** End User Tpd Limit */
end_user_tpd_limit?: number | null;
/** End User Tpm Limit */
end_user_tpm_limit?: number | null;
/** Expires */
@ -39261,10 +39320,14 @@ export interface components {
team_soft_budget?: number | null;
/** Team Spend */
team_spend?: number | null;
/** Team Tpd Limit */
team_tpd_limit?: number | null;
/** Team Tpm Limit */
team_tpm_limit?: number | null;
/** Token */
token?: string | null;
/** Tpd Limit */
tpd_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** Tpm Limit Per Model */