diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9e390312fa1..b6bef568637 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1045,6 +1045,7 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase): model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None mcp_rpm_limit: Optional[Dict[str, int]] = None + tag_rpm_limit: Optional[dict[str, int]] = None guardrails: Optional[List[str]] = None policies: Optional[List[str]] = None prompts: Optional[List[str]] = None @@ -3869,6 +3870,7 @@ LiteLLM_ManagementEndpoint_MetadataFields = [ "model_rpm_limit", "model_tpm_limit", "mcp_rpm_limit", + "tag_rpm_limit", "rpm_limit_type", "tpm_limit_type", "enforced_params", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 3c508df0cc9..893e09ece6e 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -975,6 +975,20 @@ def get_team_mcp_rpm_limit( return None +def get_key_tag_rpm_limit( + user_api_key_dict: UserAPIKeyAuth, +) -> Optional[dict[str, int]]: + """ + Get the per-request-tag rpm limit configured on a given api key. + + The returned dict is keyed by request tag, so each tag/group tracked on + the key gets its own independent RPM counter. + """ + if user_api_key_dict.metadata: + return user_api_key_dict.metadata.get("tag_rpm_limit") + return None + + def get_project_model_rpm_limit( user_api_key_dict: UserAPIKeyAuth, ) -> Optional[Dict[str, int]]: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index ee0a0e1789d..7aedb74f2ea 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -31,8 +31,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata +from litellm.proxy.auth.auth_utils import ( + get_key_tag_rpm_limit, + get_model_rate_limit_from_metadata, +) from litellm.proxy.auth.budget_throttle import throttled_limit +from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body from litellm.proxy.common_utils.proxy_rate_limit_error import ( ProxyRateLimitError, map_v3_rate_limit_type, @@ -1300,6 +1304,43 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) + def _add_tag_per_key_rate_limit_descriptor( + self, + user_api_key_dict: UserAPIKeyAuth, + data: dict, + descriptors: list[RateLimitDescriptor], + ) -> None: + """ + Add per-request-tag rpm limit descriptors for the API key. + + Each tag carried on the request that has a configured limit gets its own + ``{api_key}:{tag}`` counter, so a burst on one tag/group never consumes + another's budget. Tags without a configured limit fall through to the + key-level descriptor. + """ + if not user_api_key_dict.api_key: + return + + tag_rpm_limit = get_key_tag_rpm_limit(user_api_key_dict) or {} + if not tag_rpm_limit: + return + + for tag in dict.fromkeys(get_tags_from_request_body(data)): + rpm_limit = tag_rpm_limit.get(tag) + if rpm_limit is None: + continue + descriptors.append( + RateLimitDescriptor( + key="tag_per_key", + value=f"{user_api_key_dict.api_key}:{tag}", + rate_limit={ + "requests_per_unit": rpm_limit, + "tokens_per_unit": None, + "window_size": self.window_size, + }, + ) + ) + def _add_mcp_per_key_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, @@ -1645,6 +1686,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors=descriptors, ) + # Per-request-tag rate limits scoped to this key + self._add_tag_per_key_rate_limit_descriptor( + user_api_key_dict=user_api_key_dict, + data=data, + descriptors=descriptors, + ) + # REST MCP calls pass the raw body through this hook before server # resolution; only the later synthetic hook payload may carry this key. if call_type == CallTypes.call_mcp_tool.value and "server_id" not in data: @@ -1961,6 +2009,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # Org Level Rate Limits descriptors.extend(self.create_organization_rate_limit_descriptor(user_api_key_dict, requested_model)) + # Only check rate limits if we have descriptors with actual limits if descriptors: # First pass: RPM and max_parallel_requests sliding-window check. diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 989fad7cd0b..ccd15a68437 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -377,6 +377,7 @@ async def new_user( - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user. + - tag_rpm_limit: Optional[dict] - Per-request-tag rpm limit, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Enforced for keys only; values set on a user are stored but not enforced per user. - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - agent_id: Optional[str] - The agent id associated with the user. @@ -1379,6 +1380,7 @@ async def user_update( - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user. + - tag_rpm_limit: Optional[dict] - Per-request-tag rpm limit, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Enforced for keys only; values set on a user are stored but not enforced per user. - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - agent_id: Optional[str] - The agent id associated with the user. diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 71cf2db3dfb..63f4b731871 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1496,6 +1496,7 @@ async def generate_key_fn( - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit. - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit. - mcp_rpm_limit: Optional[dict] - key-specific per-MCP-server rpm limit, keyed by MCP server name (alias if set, else the configured name). Example - {"github": 100, "slack": 200}. IF null or {} then no MCP-specific rpm limit. + - tag_rpm_limit: Optional[dict] - key-specific per-request-tag rpm limit, keyed by request tag. Example - {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; requests whose tag is absent fall back to the key-level rpm limit. - tpm_limit_type: Optional[str] - Type of tpm limit. Options: "best_effort_throughput" (no error if we're overallocating tpm), "guaranteed_throughput" (raise an error if we're overallocating tpm), "dynamic" (dynamically exceed limit when no 429 errors). Defaults to "best_effort_throughput". - rpm_limit_type: Optional[str] - Type of rpm limit. Options: "best_effort_throughput" (no error if we're overallocating rpm), "guaranteed_throughput" (raise an error if we're overallocating rpm), "dynamic" (dynamically exceed limit when no 429 errors). Defaults to "best_effort_throughput". - allowed_cache_controls: Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request @@ -2514,6 +2515,7 @@ async def update_key_fn( - rpm_limit: Optional[int] - Requests per minute limit - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200} - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200} + - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit. - model_tpm_limit: Optional[dict] - Model-specific TPM limits {"gpt-4": 100000, "claude-v1": 200000} - tpm_limit_type: Optional[str] - TPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" - rpm_limit_type: Optional[str] - RPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" @@ -3551,6 +3553,7 @@ async def generate_key_helper_fn( model_rpm_limit: Optional[dict] = None, model_tpm_limit: Optional[dict] = None, mcp_rpm_limit: Optional[dict] = None, + tag_rpm_limit: Optional[dict] = None, guardrails: Optional[list] = None, policies: Optional[list] = None, prompts: Optional[list] = None, @@ -3624,6 +3627,9 @@ async def generate_key_helper_fn( if mcp_rpm_limit is not None: metadata = metadata or {} metadata["mcp_rpm_limit"] = mcp_rpm_limit + if tag_rpm_limit is not None: + metadata = metadata or {} + metadata["tag_rpm_limit"] = tag_rpm_limit if guardrails is not None: metadata = metadata or {} metadata["guardrails"] = guardrails diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 21a001d77d7..042fc107f40 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -19,6 +19,7 @@ from litellm.proxy.auth.auth_utils import ( get_key_mcp_rpm_limit, get_key_model_rpm_limit, get_key_model_tpm_limit, + get_key_tag_rpm_limit, get_model_from_request, get_project_model_rpm_limit, get_project_model_tpm_limit, @@ -2393,3 +2394,17 @@ class TestIsRequestBodySafeBlocksModelList: ) is True ) + + +class TestGetKeyTagRateLimits: + """Tests for get_key_tag_rpm_limit.""" + + def test_reads_tag_rpm_limit_from_metadata(self): + key = UserAPIKeyAuth( + api_key="sk-123", metadata={"tag_rpm_limit": {"cell-1": 5}} + ) + assert get_key_tag_rpm_limit(key) == {"cell-1": 5} + + def test_returns_none_when_unset(self): + key = UserAPIKeyAuth(api_key="sk-123") + assert get_key_tag_rpm_limit(key) is None diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 12f0a64a179..d150591c8de 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3573,3 +3573,136 @@ async def test_pre_call_hook_skips_reservation_when_disabled(monkeypatch): ) assert TPM_RESERVED_TOKENS_KEY not in (data.get("metadata") or {}) + + +@pytest.mark.asyncio +async def test_per_tag_rate_limit_independent_counters_v3(monkeypatch): + """ + A single key with per-tag RPM limits tracks each tag independently: a tag + at its limit returns 429 while a different (unlimited) tag keeps flowing, + governed only by the generous key-level limit. + """ + monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60") + _api_key = hash_token("sk-per-tag-rpm") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + rpm_limit=100, + metadata={"tag_rpm_limit": {"cell-1": 2}}, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + async def call(tag: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo", "metadata": {"tags": [tag]}}, + call_type="", + ) + + await call("cell-1") + await call("cell-1") + with pytest.raises(HTTPException) as exc_info: + await call("cell-1") + assert exc_info.value.status_code == 429 + assert "tag_per_key" in str(exc_info.value.detail) + + # cell-2 has no configured tag limit, so cell-1's exhausted counter must + # not block it; only the generous key-level limit applies. + for _ in range(5): + await call("cell-2") + + +@pytest.mark.asyncio +async def test_per_tag_descriptor_creation_v3(): + """ + _create_rate_limit_descriptors emits a tag_per_key descriptor carrying the + configured RPM limit only for request tags present in the configured map. + """ + _api_key = hash_token("sk-per-tag-desc") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + metadata={"tag_rpm_limit": {"cell-1": 5}}, + ) + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + + descriptors = handler._create_rate_limit_descriptors( + user_api_key_dict=user_api_key_dict, + data={"model": "gpt-3.5-turbo", "metadata": {"tags": ["cell-1", "cell-2"]}}, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + + tag_descriptors = [d for d in descriptors if d["key"] == "tag_per_key"] + assert len(tag_descriptors) == 1, "only the configured tag yields a descriptor" + descriptor = tag_descriptors[0] + assert descriptor["value"] == f"{_api_key}:cell-1" + assert descriptor["rate_limit"]["requests_per_unit"] == 5 + + +@pytest.mark.asyncio +async def test_per_tag_descriptor_absent_without_config_v3(): + """No tag_per_key descriptor is created when the key has no tag limits.""" + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-no-tag"), + rpm_limit=10, + ) + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + + descriptors = handler._create_rate_limit_descriptors( + user_api_key_dict=user_api_key_dict, + data={"model": "gpt-3.5-turbo", "metadata": {"tags": ["cell-1"]}}, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + + assert not [d for d in descriptors if d["key"] == "tag_per_key"] + + +@pytest.mark.asyncio +async def test_per_tag_untagged_request_governed_by_key_limit_v3(monkeypatch): + """ + Per-tag limits are opt-in sub-limits under the key-level ceiling, not a + standalone enforcement boundary: a request that carries no tag (or a tag + without a configured limit) is not rejected by any tag counter, but it is + still bounded by the key-level rpm_limit. This pins the documented + untagged-fallback behavior so a future "fail closed on missing tag" change + would fail here instead of silently breaking it. + """ + monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60") + _api_key = hash_token("sk-untagged-fallback") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + rpm_limit=3, + metadata={"tag_rpm_limit": {"cell-1": 2}}, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + async def call(metadata: dict) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo", "metadata": metadata}, + call_type="", + ) + + # Untagged and unconfigured-tag requests share the key-level budget of 3 + # and never hit a tag_per_key counter. + await call({}) + await call({"tags": ["cell-99"]}) + await call({}) + with pytest.raises(HTTPException) as exc_info: + await call({"tags": ["cell-99"]}) + assert exc_info.value.status_code == 429 + assert "tag_per_key" not in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 4fb3df52cf6..d707421aeb6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -15,8 +15,11 @@ from unittest.mock import AsyncMock, MagicMock, patch from fastapi import HTTPException +import inspect + from litellm.proxy._types import ( GenerateKeyRequest, + NewUserRequest, LiteLLM_BudgetTable, LiteLLM_OrganizationTable, LiteLLM_TeamTableCachedObj, @@ -14480,3 +14483,23 @@ async def test_regenerate_key_non_admin_permissions_rejected_before_enterprise_g assert int(exc.value.code) == 403 assert "permissions" in str(exc.value.message) assert "Enterprise" not in str(exc.value.message) + + +def test_generate_key_helper_fn_accepts_per_tag_rate_limits(): + """ + Regression: new_user / SSO sign-in forward NewUserRequest fields to + generate_key_helper_fn via `**data_json`. The per-tag limit field must be + an accepted kwarg, otherwise user creation 500s with + "generate_key_helper_fn() got an unexpected keyword argument 'tag_rpm_limit'". + """ + params = inspect.signature(generate_key_helper_fn).parameters + assert "tag_rpm_limit" in params + + # The field exists on the request model that new_user forwards via **data_json. + assert "tag_rpm_limit" in NewUserRequest.model_fields + + # Binding the per-tag kwarg must not raise an unexpected-keyword TypeError. + inspect.signature(generate_key_helper_fn).bind_partial( + request_type="user", + tag_rpm_limit={"cell-1": 5}, + ) diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/TagRateLimitEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/TagRateLimitEditor.tsx new file mode 100644 index 00000000000..ee022ee9a75 --- /dev/null +++ b/ui/litellm-dashboard/src/components/key_team_helpers/TagRateLimitEditor.tsx @@ -0,0 +1,103 @@ +import { Button, Input, InputNumber } from "antd"; +import React from "react"; + +export interface TagRateLimitEntry { + // Stable identity for React list keys so deleting a middle row doesn't shift + // the controlled inputs of the rows below it. + id: string; + tag: string; + rpm_limit: number | null; +} + +let nextRowId = 0; +const newRowId = (): string => `tag-row-${nextRowId++}`; + +export interface TagRateLimits { + tag_rpm_limit: Record; +} + +// Build the rpm limit map from editor rows. A tag only enters the map when its +// name is non-empty and the RPM cell holds a number. +export const tagRowsToLimits = (rows: TagRateLimitEntry[]): TagRateLimits => { + const tag_rpm_limit: Record = {}; + rows.forEach(({ tag, rpm_limit }) => { + const name = tag.trim(); + if (!name) return; + if (typeof rpm_limit === "number") tag_rpm_limit[name] = rpm_limit; + }); + return { tag_rpm_limit }; +}; + +// Coerce an untyped metadata value into a {tag: number} map, dropping anything +// that isn't a numeric entry. Key metadata is loosely typed, so validate here. +const toNumberMap = (raw: unknown): Record => { + if (!raw || typeof raw !== "object") return {}; + const out: Record = {}; + Object.entries(raw as Record).forEach(([tag, limit]) => { + if (typeof limit === "number") out[tag] = limit; + }); + return out; +}; + +// Reconstruct editor rows from the stored rpm map. +export const tagLimitsToRows = (tagRpmLimit?: unknown): TagRateLimitEntry[] => { + const rpm = toNumberMap(tagRpmLimit); + return Object.keys(rpm).map((tag) => ({ + id: newRowId(), + tag, + rpm_limit: rpm[tag], + })); +}; + +interface TagRateLimitEditorProps { + value: TagRateLimitEntry[]; + onChange: (v: TagRateLimitEntry[]) => void; +} + +export function TagRateLimitEditor({ value, onChange }: TagRateLimitEditorProps) { + const addRow = () => { + onChange([...value, { id: newRowId(), tag: "", rpm_limit: null }]); + }; + + const removeRow = (idx: number) => { + onChange(value.filter((_, i) => i !== idx)); + }; + + const updateRow = (idx: number, field: keyof TagRateLimitEntry, fieldValue: string | number | null) => { + onChange(value.map((row, i) => (i === idx ? { ...row, [field]: fieldValue } : row))); + }; + + return ( +
+ {value.map((row, idx) => ( +
+ updateRow(idx, "tag", e.target.value)} + placeholder="Tag (e.g. cell-1)" + style={{ width: 180 }} + /> + updateRow(idx, "rpm_limit", v ?? null)} + placeholder="RPM" + style={{ width: 120 }} + /> + +
+ ))} + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 0f371b72efe..ef2ddab70ed 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -30,6 +30,7 @@ import ProjectDropdown from "../common_components/ProjectDropdown"; import { CreateUserButton } from "../CreateUserButton"; import { BudgetFallbacksEditor } from "../key_team_helpers/BudgetFallbacksEditor"; import { BudgetWindowEntry, BudgetWindowsEditor } from "../key_team_helpers/BudgetWindowsEditor"; +import { TagRateLimitEditor, TagRateLimitEntry, tagRowsToLimits } from "../key_team_helpers/TagRateLimitEditor"; import { excludeProxyWideSentinel, getModelDisplayName, @@ -202,6 +203,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const [rotationInterval, setRotationInterval] = useState("30d"); const [routerSettings, setRouterSettings] = useState(null); const [budgetLimits, setBudgetLimits] = useState([]); + const [tagRateLimits, setTagRateLimits] = useState([]); const [budgetFallbacks, setBudgetFallbacks] = useState>({}); const [budgetFallbacksKey, setBudgetFallbacksKey] = useState(0); const [routerSettingsKey, setRouterSettingsKey] = useState(0); @@ -223,6 +225,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp setSelectedOrganizationId(null); setSelectedProjectId(null); setBudgetLimits([]); + setTagRateLimits([]); setBudgetFallbacks({}); setBudgetFallbacksKey((k) => k + 1); }; @@ -244,6 +247,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp setSelectedOrganizationId(null); setSelectedProjectId(null); setBudgetLimits([]); + setTagRateLimits([]); setBudgetFallbacks({}); setBudgetFallbacksKey((k) => k + 1); }; @@ -543,6 +547,12 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp formValues.budget_limits = validWindows; } + // Add per-tag rate limits (only when at least one row is configured) + const { tag_rpm_limit } = tagRowsToLimits(tagRateLimits); + if (Object.keys(tag_rpm_limit).length > 0) { + formValues.tag_rpm_limit = tag_rpm_limit; + } + if (Object.keys(budgetFallbacks).length > 0) { formValues.budget_fallbacks = budgetFallbacks; } @@ -567,6 +577,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp NotificationsManager.success("Virtual Key Created"); form.resetFields(); setBudgetLimits([]); + setTagRateLimits([]); setBudgetFallbacks({}); setBudgetFallbacksKey((k) => k + 1); localStorage.removeItem("userData" + userID); @@ -1177,6 +1188,19 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp form={form} showDetailedDescriptions={true} /> + + Per-Tag Rate Limits{" "} + + + + + } + > + + ( Array.isArray(keyData.budget_limits) ? keyData.budget_limits : [], ); + const [tagRateLimits, setTagRateLimits] = useState( + tagLimitsToRows(keyData.metadata?.tag_rpm_limit), + ); const [budgetFallbacks, setBudgetFallbacks] = useState>( keyData.budget_fallbacks && typeof keyData.budget_fallbacks === "object" ? keyData.budget_fallbacks : {}, ); @@ -311,6 +320,11 @@ export function KeyEditView({ values.budget_limits = []; } + // Always send the current per-tag limit map so removing every row + // clears the stored limits ({} overwrites the metadata field). + const { tag_rpm_limit } = tagRowsToLimits(tagRateLimits); + values.tag_rpm_limit = tag_rpm_limit; + const hadExistingFallbacks = keyData.budget_fallbacks != null && Object.keys(keyData.budget_fallbacks).length > 0; if (Object.keys(budgetFallbacks).length > 0) { values.budget_fallbacks = budgetFallbacks; @@ -553,6 +567,19 @@ export function KeyEditView({ + + Per-Tag Rate Limits{" "} + + + + + } + > + + + {accessToken && ( + + Tag RPM Limits:{" "} + {currentKeyData.metadata?.tag_rpm_limit && + Object.keys(currentKeyData.metadata.tag_rpm_limit).length > 0 + ? JSON.stringify(currentKeyData.metadata.tag_rpm_limit) + : "Unlimited"} +
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d9bba85bf4b..23ada336710 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6509,6 +6509,7 @@ export interface paths { * - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit. * - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit. * - mcp_rpm_limit: Optional[dict] - key-specific per-MCP-server rpm limit, keyed by MCP server name (alias if set, else the configured name). Example - {"github": 100, "slack": 200}. IF null or {} then no MCP-specific rpm limit. + * - tag_rpm_limit: Optional[dict] - key-specific per-request-tag rpm limit, keyed by request tag. Example - {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; requests whose tag is absent fall back to the key-level rpm limit. * - tpm_limit_type: Optional[str] - Type of tpm limit. Options: "best_effort_throughput" (no error if we're overallocating tpm), "guaranteed_throughput" (raise an error if we're overallocating tpm), "dynamic" (dynamically exceed limit when no 429 errors). Defaults to "best_effort_throughput". * - rpm_limit_type: Optional[str] - Type of rpm limit. Options: "best_effort_throughput" (no error if we're overallocating rpm), "guaranteed_throughput" (raise an error if we're overallocating rpm), "dynamic" (dynamically exceed limit when no 429 errors). Defaults to "best_effort_throughput". * - allowed_cache_controls: Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request @@ -6896,6 +6897,7 @@ export interface paths { * - rpm_limit: Optional[int] - Requests per minute limit * - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200} * - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200} + * - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit. * - model_tpm_limit: Optional[dict] - Model-specific TPM limits {"gpt-4": 100000, "claude-v1": 200000} * - tpm_limit_type: Optional[str] - TPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" * - rpm_limit_type: Optional[str] - RPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" @@ -14623,6 +14625,7 @@ export interface paths { * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) * - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user. + * - tag_rpm_limit: Optional[dict] - Per-request-tag rpm limit, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Enforced for keys only; values set on a user are stored but not enforced per user. * - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) * - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). * - agent_id: Optional[str] - The agent id associated with the user. @@ -14704,6 +14707,7 @@ export interface paths { * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) * - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user. + * - tag_rpm_limit: Optional[dict] - Per-request-tag rpm limit, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Enforced for keys only; values set on a user are stored but not enforced per user. * - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) * - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). * - agent_id: Optional[str] - The agent id associated with the user. @@ -23705,6 +23709,10 @@ export interface components { * @default 0 */ spend: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Tags */ tags?: string[] | null; /** Team Id */ @@ -23847,6 +23855,10 @@ export interface components { * @default 0 */ spend: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Tags */ tags?: string[] | null; /** Team Id */ @@ -28032,6 +28044,10 @@ export interface components { spend: number | null; /** Sso User Id */ sso_user_id?: string | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Team Id */ team_id?: string | null; /** Teams */ @@ -28186,6 +28202,10 @@ export interface components { * @default 0 */ spend: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Tags */ tags?: string[] | null; /** Team Id */ @@ -29817,6 +29837,10 @@ export interface components { soft_budget?: number | null; /** Spend */ spend?: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Tags */ tags?: string[] | null; /** Team Id */ @@ -31762,6 +31786,10 @@ export interface components { rpm_limit_type?: ("guaranteed_throughput" | "best_effort_throughput" | "dynamic") | null; /** Spend */ spend?: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Tags */ tags?: string[] | null; /** Team Id */ @@ -32218,6 +32246,10 @@ export interface components { rpm_limit?: number | null; /** Spend */ spend?: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Team Id */ team_id?: string | null; /** Tpm Limit */ @@ -32320,6 +32352,10 @@ export interface components { rpm_limit?: number | null; /** Spend */ spend?: number | null; + /** Tag Rpm Limit */ + tag_rpm_limit?: { + [key: string]: number; + } | null; /** Team Id */ team_id?: string | null; /** Tpm Limit */