diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 574a5d60d3c..d81f2cd0b3e 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -36,7 +36,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject -from litellm.types.utils import ModelResponse, Usage +from litellm.types.utils import CallTypes, ModelResponse, Usage if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -1606,6 +1606,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): rpm_limit_type: Optional[str], tpm_limit_type: Optional[str], model_has_failures: bool, + call_type: Optional[str] = None, ) -> List[RateLimitDescriptor]: """ Create all rate limit descriptors for the request. @@ -1726,18 +1727,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors=descriptors, ) - # Per-MCP-server rate limits - mcp_server_name = data.get("mcp_server_name", None) - self._add_mcp_per_key_rate_limit_descriptor( - user_api_key_dict=user_api_key_dict, - mcp_server_name=mcp_server_name, - descriptors=descriptors, - ) - self._add_mcp_per_team_rate_limit_descriptor( - user_api_key_dict=user_api_key_dict, - mcp_server_name=mcp_server_name, - descriptors=descriptors, - ) + # Per-MCP-server rate limits. Only honor mcp_server_name on actual MCP + # tool calls; otherwise a normal LLM request could inject it in its body + # to consume another server's MCP quota and 429 legitimate tool calls. + if call_type == CallTypes.call_mcp_tool.value: + mcp_server_name = data.get("mcp_server_name", None) + self._add_mcp_per_key_rate_limit_descriptor( + user_api_key_dict=user_api_key_dict, + mcp_server_name=mcp_server_name, + descriptors=descriptors, + ) + self._add_mcp_per_team_rate_limit_descriptor( + user_api_key_dict=user_api_key_dict, + mcp_server_name=mcp_server_name, + descriptors=descriptors, + ) if ( get_team_model_rpm_limit(user_api_key_dict) is not None @@ -2069,6 +2073,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): rpm_limit_type=rpm_limit_type, tpm_limit_type=tpm_limit_type, model_has_failures=model_has_failures, + call_type=call_type, ) # Add team model rate limits from team_metadata diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 75eb5cd55ef..4fd28b0f6eb 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -386,6 +386,7 @@ async def new_user( - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) + - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200} - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - agent_id: Optional[str] - The agent id associated with the user. @@ -1427,6 +1428,7 @@ async def user_update( - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) + - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200} - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - agent_id: Optional[str] - The agent id associated with the user. diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 1714efd7d1a..93b552e6f4c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2424,6 +2424,7 @@ async def update_key_fn( # noqa: PLR0915 - tpm_limit: Optional[int] - Tokens per minute limit - rpm_limit: Optional[int] - Requests per minute limit - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200} + - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200} - model_tpm_limit: Optional[dict] - Model-specific TPM limits {"gpt-4": 100000, "claude-v1": 200000} - tpm_limit_type: Optional[str] - TPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" - rpm_limit_type: Optional[str] - RPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 85fb78a95e2..c16b8955d61 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -2910,13 +2910,14 @@ def _find_descriptor(descriptors, key): return next((d for d in descriptors if d["key"] == key), None) -def _build_mcp_descriptors(handler, user_api_key_dict, data): +def _build_mcp_descriptors(handler, user_api_key_dict, data, call_type="call_mcp_tool"): return handler._create_rate_limit_descriptors( user_api_key_dict=user_api_key_dict, data=data, rpm_limit_type=None, tpm_limit_type=None, model_has_failures=False, + call_type=call_type, ) @@ -2956,13 +2957,21 @@ def test_mcp_per_key_descriptor_skipped_for_non_matching_server_v3(): def test_mcp_descriptor_skipped_for_non_mcp_request_v3(): + """A non-MCP request must not create an MCP descriptor even if the caller + injects mcp_server_name in the body; otherwise an LLM call could consume a + target server's MCP quota and 429 legitimate tool calls.""" handler, _ = _make_mcp_handler() user_api_key_dict = UserAPIKeyAuth( api_key=hash_token("sk-mcp-key"), metadata={"mcp_rpm_limit": {"github": 5}}, ) - descriptors = _build_mcp_descriptors(handler, user_api_key_dict, {"model": "gpt-4"}) + descriptors = _build_mcp_descriptors( + handler, + user_api_key_dict, + {"model": "gpt-4", "mcp_server_name": "github"}, + call_type="completion", + ) assert _find_descriptor(descriptors, "mcp_per_key") is None