From f8bf45aae0ad214941fda6a86a43ebda9353cd0a Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 21 Oct 2026 12:15:40 +0000 Subject: [PATCH] fix(mcp): gate the full-body charge on the limiter's effective descriptors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/server.py | 24 ++-- .../hooks/parallel_request_limiter_v3.py | 21 +++- .../mcp_server/test_mcp_server.py | 109 +++++++++++++++++- .../hooks/test_parallel_request_limiter_v3.py | 15 +++ 4 files changed, 151 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index f8645fc4884..c0b90570ee9 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -136,18 +136,6 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" _DECODED_JSON: Final = TypeAdapter(object) -def _has_request_rate_limit(user_api_key_auth: UserAPIKeyAuth) -> bool: - return ( - user_api_key_auth.rpm_limit is not None - or user_api_key_auth.max_parallel_requests is not None - or user_api_key_auth.user_rpm_limit is not None - or user_api_key_auth.team_rpm_limit is not None - or user_api_key_auth.team_member_rpm_limit is not None - or user_api_key_auth.organization_rpm_limit is not None - or user_api_key_auth.end_user_rpm_limit is not None - ) - - def _mcp_setup_request_capacity( method: str | None, user_api_key_auth: UserAPIKeyAuth | None, @@ -164,6 +152,16 @@ def _mcp_setup_request_capacity( return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={}) +async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool: + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import proxy_logging_obj + + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return False + return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call") + + def reject_disallowed_mcp_origin(request: StarletteRequest) -> None: from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup @@ -2737,7 +2735,7 @@ if MCP_AVAILABLE: if mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES - and _has_request_rate_limit(user_api_key_auth) + and await _mcp_setup_request_limit_applies(user_api_key_auth) else mcp_method ) request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..f7430456285 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): *await self._create_tag_rate_limit_descriptors(data), ] + @staticmethod + def _request_capacity_data( + model: str, + request_data: Mapping[str, object] | None = None, + ) -> Mapping[str, object]: + return MappingProxyType({**(request_data or MappingProxyType({})), "model": model}) + + async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: + data: Final = self._request_capacity_data(model) + descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) + return any( + descriptor["rate_limit"] is not None + and ( + descriptor["rate_limit"].get("requests_per_unit") is not None + or descriptor["rate_limit"].get("max_parallel_requests") is not None + ) + for descriptor in descriptors + ) + async def _release_request_capacity_when_admitted( self, admission: asyncio.Task[RateLimitResponse], @@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): request_data: Mapping[str, object] | None = None, ) -> AsyncGenerator[None, None]: """Charge one non-generation provider request to RPM and hold its concurrency slot.""" - data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model}) + data: Final = self._request_capacity_data(model, request_data) descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) acquisition: Final = ParallelSlotAcquisition( slot_id=uuid.uuid4().hex, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index d33dc412b4c..c0d8edf1a50 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -11,7 +11,7 @@ from contextlib import asynccontextmanager, nullcontext from starlette.types import Receive, Scope, Send - +from litellm.types.agents import AgentResponse from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, MCPServer, @@ -3265,7 +3265,8 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): async def _streamable_mcp_request_runner( monkeypatch: pytest.MonkeyPatch, auth: UserAPIKeyAuth, -) -> AsyncIterator[tuple[Callable[[str], Awaitable[None]], AsyncMock]]: + on_receive_count_at_dispatch: Callable[[int], None] | None = None, +) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]: from litellm.caching.dual_cache import DualCache from litellm.proxy import proxy_server from litellm.proxy._experimental.mcp_server import server @@ -3299,7 +3300,11 @@ async def _streamable_mcp_request_runner( monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope) - async def request(method: str, request_body: bytes | None = None) -> None: + async def request( + method: str, + request_body: bytes | None = None, + request_chunks: tuple[bytes, ...] | None = None, + ) -> None: body: Final = ( request_body if request_body is not None @@ -3320,13 +3325,41 @@ async def _streamable_mcp_request_runner( "server": ("localhost", 4000), "scheme": "http", } - receive: Final[Receive] = AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}) + request_messages: Final = ( + tuple( + { + "type": "http.request", + "body": chunk, + "more_body": index < len(request_chunks) - 1, + } + for index, chunk in enumerate(request_chunks) + ) + if request_chunks is not None + else ({"type": "http.request", "body": body, "more_body": False},) + ) + receive: Final[Receive] = AsyncMock(side_effect=request_messages) + if on_receive_count_at_dispatch is not None: + def observe_receive_count(*_args: object, **_kwargs: object) -> None: + on_receive_count_at_dispatch(receive.await_count) + + dispatch.side_effect = observe_receive_count send: Final[Send] = AsyncMock() await server.handle_streamable_http_mcp(scope, receive, send) yield request, dispatch +def _padded_tools_list_body() -> bytes: + return json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + + @pytest.mark.asyncio async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError @@ -3368,6 +3401,54 @@ async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limi assert dispatch.await_count == 1 +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth( + api_key="mcp-oversized-model-rpm-key", + metadata={"model_rpm_limit": {"mcp-tool-call": 1}}, + ) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = _padded_tools_list_body() + await request("tools/list", oversized_body) + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + agent_id: Final = "mcp-setup-agent-rpm" + agent_policy: Final = AgentResponse( + agent_id=agent_id, + agent_name="mcp-setup-agent", + agent_card_params={"name": "mcp-setup-agent"}, + rpm_limit=1, + ) + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy( + update={"managed_agent_policy": agent_policy} + ) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = _padded_tools_list_body() + await request("tools/list", oversized_body) + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + @pytest.mark.asyncio async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits( monkeypatch: pytest.MonkeyPatch, @@ -3387,6 +3468,26 @@ async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual assert dispatch.await_count == 1 +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._experimental.mcp_server import server as mcp_server + + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key") + + def assert_receive_count(receive_count: int) -> None: + assert receive_count == 1 + + async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch): + assert await mcp_server._mcp_setup_request_limit_applies(auth) is False + oversized_body: Final = _padded_tools_list_body() + peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES + await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:])) + + assert dispatch.await_count == 1 + + @pytest.mark.asyncio async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None: auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 1ec505362dd..4de06e42e43 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes( assert counter_scope in str(exc.value.detail) +@pytest.mark.asyncio +async def test_has_request_capacity_limit_matches_effective_request_descriptors(): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit")) + key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1) + model_rpm_auth = UserAPIKeyAuth( + api_key=hash_token("sk-count-model-request-limit"), + metadata={"model_rpm_limit": {"mcp-tool-call": 1}}, + ) + + assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False + assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True + assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True + + @pytest.mark.asyncio async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch): import litellm.proxy.proxy_server as proxy_server