mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(mcp): gate the full-body charge on the limiter's effective descriptors
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
36b18ceecf
commit
f8bf45aae0
4 changed files with 151 additions and 18 deletions
|
|
@ -136,18 +136,6 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version"
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _has_request_rate_limit(user_api_key_auth: UserAPIKeyAuth) -> bool:
|
||||
return (
|
||||
user_api_key_auth.rpm_limit is not None
|
||||
or user_api_key_auth.max_parallel_requests is not None
|
||||
or user_api_key_auth.user_rpm_limit is not None
|
||||
or user_api_key_auth.team_rpm_limit is not None
|
||||
or user_api_key_auth.team_member_rpm_limit is not None
|
||||
or user_api_key_auth.organization_rpm_limit is not None
|
||||
or user_api_key_auth.end_user_rpm_limit is not None
|
||||
)
|
||||
|
||||
|
||||
def _mcp_setup_request_capacity(
|
||||
method: str | None,
|
||||
user_api_key_auth: UserAPIKeyAuth | None,
|
||||
|
|
@ -164,6 +152,16 @@ def _mcp_setup_request_capacity(
|
|||
return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={})
|
||||
|
||||
|
||||
async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool:
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
|
||||
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
|
||||
return False
|
||||
return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call")
|
||||
|
||||
|
||||
def reject_disallowed_mcp_origin(request: StarletteRequest) -> None:
|
||||
from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup
|
||||
|
||||
|
|
@ -2737,7 +2735,7 @@ if MCP_AVAILABLE:
|
|||
if mcp_method is None
|
||||
and peek is not None
|
||||
and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES
|
||||
and _has_request_rate_limit(user_api_key_auth)
|
||||
and await _mcp_setup_request_limit_applies(user_api_key_auth)
|
||||
else mcp_method
|
||||
)
|
||||
request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth)
|
||||
|
|
|
|||
|
|
@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
*await self._create_tag_rate_limit_descriptors(data),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _request_capacity_data(
|
||||
model: str,
|
||||
request_data: Mapping[str, object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
return MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
|
||||
|
||||
async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool:
|
||||
data: Final = self._request_capacity_data(model)
|
||||
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
|
||||
return any(
|
||||
descriptor["rate_limit"] is not None
|
||||
and (
|
||||
descriptor["rate_limit"].get("requests_per_unit") is not None
|
||||
or descriptor["rate_limit"].get("max_parallel_requests") is not None
|
||||
)
|
||||
for descriptor in descriptors
|
||||
)
|
||||
|
||||
async def _release_request_capacity_when_admitted(
|
||||
self,
|
||||
admission: asyncio.Task[RateLimitResponse],
|
||||
|
|
@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
request_data: Mapping[str, object] | None = None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Charge one non-generation provider request to RPM and hold its concurrency slot."""
|
||||
data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
|
||||
data: Final = self._request_capacity_data(model, request_data)
|
||||
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
|
||||
acquisition: Final = ParallelSlotAcquisition(
|
||||
slot_id=uuid.uuid4().hex,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from contextlib import asynccontextmanager, nullcontext
|
|||
|
||||
from starlette.types import Receive, Scope, Send
|
||||
|
||||
|
||||
from litellm.types.agents import AgentResponse
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
MCPServerManager,
|
||||
MCPServer,
|
||||
|
|
@ -3265,7 +3265,8 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth():
|
|||
async def _streamable_mcp_request_runner(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
auth: UserAPIKeyAuth,
|
||||
) -> AsyncIterator[tuple[Callable[[str], Awaitable[None]], AsyncMock]]:
|
||||
on_receive_count_at_dispatch: Callable[[int], None] | None = None,
|
||||
) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._experimental.mcp_server import server
|
||||
|
|
@ -3299,7 +3300,11 @@ async def _streamable_mcp_request_runner(
|
|||
|
||||
monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope)
|
||||
|
||||
async def request(method: str, request_body: bytes | None = None) -> None:
|
||||
async def request(
|
||||
method: str,
|
||||
request_body: bytes | None = None,
|
||||
request_chunks: tuple[bytes, ...] | None = None,
|
||||
) -> None:
|
||||
body: Final = (
|
||||
request_body
|
||||
if request_body is not None
|
||||
|
|
@ -3320,13 +3325,41 @@ async def _streamable_mcp_request_runner(
|
|||
"server": ("localhost", 4000),
|
||||
"scheme": "http",
|
||||
}
|
||||
receive: Final[Receive] = AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False})
|
||||
request_messages: Final = (
|
||||
tuple(
|
||||
{
|
||||
"type": "http.request",
|
||||
"body": chunk,
|
||||
"more_body": index < len(request_chunks) - 1,
|
||||
}
|
||||
for index, chunk in enumerate(request_chunks)
|
||||
)
|
||||
if request_chunks is not None
|
||||
else ({"type": "http.request", "body": body, "more_body": False},)
|
||||
)
|
||||
receive: Final[Receive] = AsyncMock(side_effect=request_messages)
|
||||
if on_receive_count_at_dispatch is not None:
|
||||
def observe_receive_count(*_args: object, **_kwargs: object) -> None:
|
||||
on_receive_count_at_dispatch(receive.await_count)
|
||||
|
||||
dispatch.side_effect = observe_receive_count
|
||||
send: Final[Send] = AsyncMock()
|
||||
await server.handle_streamable_http_mcp(scope, receive, send)
|
||||
|
||||
yield request, dispatch
|
||||
|
||||
|
||||
def _padded_tools_list_body() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"params": {"_meta": {"pad": "x" * 5000}},
|
||||
"method": "tools/list",
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
|
@ -3368,6 +3401,54 @@ async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limi
|
|||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key="mcp-oversized-model-rpm-key",
|
||||
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
|
||||
)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
oversized_body: Final = _padded_tools_list_body()
|
||||
await request("tools/list", oversized_body)
|
||||
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
|
||||
await request("tools/list", oversized_body)
|
||||
|
||||
assert rate_limit_error.value.status_code == 429
|
||||
assert rate_limit_error.value.headers.get("retry-after") is not None
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
agent_id: Final = "mcp-setup-agent-rpm"
|
||||
agent_policy: Final = AgentResponse(
|
||||
agent_id=agent_id,
|
||||
agent_name="mcp-setup-agent",
|
||||
agent_card_params={"name": "mcp-setup-agent"},
|
||||
rpm_limit=1,
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy(
|
||||
update={"managed_agent_policy": agent_policy}
|
||||
)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
oversized_body: Final = _padded_tools_list_body()
|
||||
await request("tools/list", oversized_body)
|
||||
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
|
||||
await request("tools/list", oversized_body)
|
||||
|
||||
assert rate_limit_error.value.status_code == 429
|
||||
assert rate_limit_error.value.headers.get("retry-after") is not None
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -3387,6 +3468,26 @@ async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual
|
|||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy._experimental.mcp_server import server as mcp_server
|
||||
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key")
|
||||
|
||||
def assert_receive_count(receive_count: int) -> None:
|
||||
assert receive_count == 1
|
||||
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch):
|
||||
assert await mcp_server._mcp_setup_request_limit_applies(auth) is False
|
||||
oversized_body: Final = _padded_tools_list_body()
|
||||
peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES
|
||||
await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:]))
|
||||
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1)
|
||||
|
|
|
|||
|
|
@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes(
|
|||
assert counter_scope in str(exc.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_has_request_capacity_limit_matches_effective_request_descriptors():
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit"))
|
||||
key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1)
|
||||
model_rpm_auth = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-count-model-request-limit"),
|
||||
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
|
||||
)
|
||||
|
||||
assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False
|
||||
assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True
|
||||
assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue