fix(mcp): gate the full-body charge on the limiter's effective descriptors

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-10-21 12:15:40 +00:00
parent 36b18ceecf
commit f8bf45aae0
4 changed files with 151 additions and 18 deletions

View file

@ -136,18 +136,6 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version"
_DECODED_JSON: Final = TypeAdapter(object)
def _has_request_rate_limit(user_api_key_auth: UserAPIKeyAuth) -> bool:
return (
user_api_key_auth.rpm_limit is not None
or user_api_key_auth.max_parallel_requests is not None
or user_api_key_auth.user_rpm_limit is not None
or user_api_key_auth.team_rpm_limit is not None
or user_api_key_auth.team_member_rpm_limit is not None
or user_api_key_auth.organization_rpm_limit is not None
or user_api_key_auth.end_user_rpm_limit is not None
)
def _mcp_setup_request_capacity(
method: str | None,
user_api_key_auth: UserAPIKeyAuth | None,
@ -164,6 +152,16 @@ def _mcp_setup_request_capacity(
return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={})
async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool:
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
from litellm.proxy.proxy_server import proxy_logging_obj
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
return False
return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call")
def reject_disallowed_mcp_origin(request: StarletteRequest) -> None:
from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup
@ -2737,7 +2735,7 @@ if MCP_AVAILABLE:
if mcp_method is None
and peek is not None
and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES
and _has_request_rate_limit(user_api_key_auth)
and await _mcp_setup_request_limit_applies(user_api_key_auth)
else mcp_method
)
request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth)

View file

@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
*await self._create_tag_rate_limit_descriptors(data),
]
@staticmethod
def _request_capacity_data(
model: str,
request_data: Mapping[str, object] | None = None,
) -> Mapping[str, object]:
return MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool:
data: Final = self._request_capacity_data(model)
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
return any(
descriptor["rate_limit"] is not None
and (
descriptor["rate_limit"].get("requests_per_unit") is not None
or descriptor["rate_limit"].get("max_parallel_requests") is not None
)
for descriptor in descriptors
)
async def _release_request_capacity_when_admitted(
self,
admission: asyncio.Task[RateLimitResponse],
@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
request_data: Mapping[str, object] | None = None,
) -> AsyncGenerator[None, None]:
"""Charge one non-generation provider request to RPM and hold its concurrency slot."""
data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
data: Final = self._request_capacity_data(model, request_data)
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
acquisition: Final = ParallelSlotAcquisition(
slot_id=uuid.uuid4().hex,

View file

@ -11,7 +11,7 @@ from contextlib import asynccontextmanager, nullcontext
from starlette.types import Receive, Scope, Send
from litellm.types.agents import AgentResponse
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
MCPServerManager,
MCPServer,
@ -3265,7 +3265,8 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth():
async def _streamable_mcp_request_runner(
monkeypatch: pytest.MonkeyPatch,
auth: UserAPIKeyAuth,
) -> AsyncIterator[tuple[Callable[[str], Awaitable[None]], AsyncMock]]:
on_receive_count_at_dispatch: Callable[[int], None] | None = None,
) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]:
from litellm.caching.dual_cache import DualCache
from litellm.proxy import proxy_server
from litellm.proxy._experimental.mcp_server import server
@ -3299,7 +3300,11 @@ async def _streamable_mcp_request_runner(
monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope)
async def request(method: str, request_body: bytes | None = None) -> None:
async def request(
method: str,
request_body: bytes | None = None,
request_chunks: tuple[bytes, ...] | None = None,
) -> None:
body: Final = (
request_body
if request_body is not None
@ -3320,13 +3325,41 @@ async def _streamable_mcp_request_runner(
"server": ("localhost", 4000),
"scheme": "http",
}
receive: Final[Receive] = AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False})
request_messages: Final = (
tuple(
{
"type": "http.request",
"body": chunk,
"more_body": index < len(request_chunks) - 1,
}
for index, chunk in enumerate(request_chunks)
)
if request_chunks is not None
else ({"type": "http.request", "body": body, "more_body": False},)
)
receive: Final[Receive] = AsyncMock(side_effect=request_messages)
if on_receive_count_at_dispatch is not None:
def observe_receive_count(*_args: object, **_kwargs: object) -> None:
on_receive_count_at_dispatch(receive.await_count)
dispatch.side_effect = observe_receive_count
send: Final[Send] = AsyncMock()
await server.handle_streamable_http_mcp(scope, receive, send)
yield request, dispatch
def _padded_tools_list_body() -> bytes:
return json.dumps(
{
"jsonrpc": "2.0",
"id": 2,
"params": {"_meta": {"pad": "x" * 5000}},
"method": "tools/list",
}
).encode()
@pytest.mark.asyncio
async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
@ -3368,6 +3401,54 @@ async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limi
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
auth: Final = UserAPIKeyAuth(
api_key="mcp-oversized-model-rpm-key",
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
oversized_body: Final = _padded_tools_list_body()
await request("tools/list", oversized_body)
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
await request("tools/list", oversized_body)
assert rate_limit_error.value.status_code == 429
assert rate_limit_error.value.headers.get("retry-after") is not None
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
agent_id: Final = "mcp-setup-agent-rpm"
agent_policy: Final = AgentResponse(
agent_id=agent_id,
agent_name="mcp-setup-agent",
agent_card_params={"name": "mcp-setup-agent"},
rpm_limit=1,
)
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy(
update={"managed_agent_policy": agent_policy}
)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
oversized_body: Final = _padded_tools_list_body()
await request("tools/list", oversized_body)
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
await request("tools/list", oversized_body)
assert rate_limit_error.value.status_code == 429
assert rate_limit_error.value.headers.get("retry-after") is not None
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits(
monkeypatch: pytest.MonkeyPatch,
@ -3387,6 +3468,26 @@ async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy._experimental.mcp_server import server as mcp_server
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key")
def assert_receive_count(receive_count: int) -> None:
assert receive_count == 1
async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch):
assert await mcp_server._mcp_setup_request_limit_applies(auth) is False
oversized_body: Final = _padded_tools_list_body()
peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES
await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:]))
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None:
auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1)

View file

@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes(
assert counter_scope in str(exc.value.detail)
@pytest.mark.asyncio
async def test_has_request_capacity_limit_matches_effective_request_descriptors():
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit"))
key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1)
model_rpm_auth = UserAPIKeyAuth(
api_key=hash_token("sk-count-model-request-limit"),
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
)
assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False
assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True
assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True
@pytest.mark.asyncio
async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch):
import litellm.proxy.proxy_server as proxy_server