diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 7207c4150ad..2ddde52f08c 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -115,6 +115,7 @@ if TYPE_CHECKING: _redact_mcp_resource_url: Final = redact_mcp_resource_url +_MCP_SETUP_REQUEST_METHODS: Final = frozenset({"initialize", "tools/list"}) _STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS: Final = 30 * 60 # Upper bound on concurrent stateful sessions a single caller may hold. Each # `initialize` creates a session that survives until the idle timeout, so @@ -135,6 +136,32 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" _DECODED_JSON: Final = TypeAdapter(object) +def _mcp_setup_request_capacity( + method: str | None, + user_api_key_auth: UserAPIKeyAuth | None, +) -> contextlib.AbstractAsyncContextManager[None]: + if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None: + return contextlib.nullcontext() + + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import proxy_logging_obj + + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return contextlib.nullcontext() + return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={}) + + +async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool: + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import proxy_logging_obj + + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return False + return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call") + + def reject_disallowed_mcp_origin(request: StarletteRequest) -> None: from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup @@ -1403,18 +1430,15 @@ if MCP_AVAILABLE: return b"{}" return json.dumps({"method": method}).encode() - def _is_initialize_request(body: bytes) -> bool: - """ - Check if the request body is a JSON-RPC initialize method. - Returns True if method is "initialize", False otherwise or on parse error. - """ + def _mcp_request_method(body: bytes) -> str | None: if not body: - return False + return None try: data: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(body))) - return data is not None and data.get("method") == "initialize" + method: Final = data.get("method") if data is not None else None + return method if isinstance(method, str) else None except (json.JSONDecodeError, UnicodeDecodeError, TypeError): - return False + return None def _jsonrpc_request_method(body: bytes) -> str | None: try: @@ -2433,7 +2457,8 @@ if MCP_AVAILABLE: ) = await extract_mcp_auth_context(scope, path) admit_mcp_request(scope, user_api_key_auth, mcp_servers) body: Final = await peek.body() if peek is not None else b"" - is_initialize: Final = _is_initialize_request(body) + mcp_method: Final = _mcp_request_method(body) + is_initialize: Final = mcp_method == "initialize" scoped_server_endpoint: Final = len(_get_mcp_servers_in_path(path) or []) == 1 request_headers: Final = StarletteRequest(scope).headers defer_upstream_probes: Final = request_headers.get( @@ -2715,11 +2740,20 @@ if MCP_AVAILABLE: if use_stateful and session_id and scope.get("method") == "DELETE": _remove_stateful_session_tracking(session_id) + charged_method: Final = ( + _jsonrpc_request_method(await peek.full_body()) + if mcp_method is None + and peek is not None + and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES + and await _mcp_setup_request_limit_applies(user_api_key_auth) + else mcp_method + ) + request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth) try: - if session_lock is not None: - async with session_lock: - await _dispatch() - else: + async with ( + session_lock if session_lock is not None else contextlib.nullcontext(), + request_capacity, + ): await _dispatch() finally: for active_request_session_id in active_request_session_ids: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 567b8452ae2..dd98f49ddd6 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): *await self._create_tag_rate_limit_descriptors(data), ] + @staticmethod + def _request_capacity_data( + model: str, + request_data: Mapping[str, object] | None = None, + ) -> Mapping[str, object]: + return MappingProxyType({**(request_data or MappingProxyType({})), "model": model}) + + async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: + data: Final = self._request_capacity_data(model) + descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) + return any( + descriptor["rate_limit"] is not None + and ( + descriptor["rate_limit"].get("requests_per_unit") is not None + or descriptor["rate_limit"].get("max_parallel_requests") is not None + ) + for descriptor in descriptors + ) + async def _release_request_capacity_when_admitted( self, admission: asyncio.Task[RateLimitResponse], @@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): request_data: Mapping[str, object] | None = None, ) -> AsyncGenerator[None, None]: """Charge one non-generation provider request to RPM and hold its concurrency slot.""" - data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model}) + data: Final = self._request_capacity_data(model, request_data) descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) acquisition: Final = ParallelSlotAcquisition( slot_id=uuid.uuid4().hex, diff --git a/tests/integration/mcp/test_mcp_rate_limits.py b/tests/integration/mcp/test_mcp_rate_limits.py index 273419d624f..b872bd5c740 100644 --- a/tests/integration/mcp/test_mcp_rate_limits.py +++ b/tests/integration/mcp/test_mcp_rate_limits.py @@ -269,3 +269,83 @@ def test_mcp_key_team_and_server_rpm_limits_share_redis(tmp_path: Path, monkeypa server_rejected: Final = _call_tool(second_replica, key_three, server_id, "rpm-add") _assert_rate_limit(server_rejected, "mcp_server") assert tool_calls(peer.drain()) == () + + +def test_streamable_mcp_setup_requests_use_virtual_key_rpm_limit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + results_directory: Final = tmp_path / "results" + results_directory.mkdir() + monkeypatch.setenv("INTEGRATION_RESULTS_DIR", str(results_directory)) + assert os.environ.get("DATABASE_URL"), "This integration case requires disposable-database access" + + with ( + owned_redis(tmp_path) as redis, + scratch_database() as database_url, + mcp_peer() as peer, + httpx.Client() as client, + ): + monkeypatch.setenv("DATABASE_URL", database_url) + seed: Final = Gateway(client, "sk-mcp-streamable-setup-rate-limit", peer.url) + config: Final = _config_file( + tmp_path, + seed.key, + redis, + upstream=peer.url, + store_model_in_db=True, + ) + environment: Final = { + "DATABASE_URL": database_url, + "LITELLM_SALT_KEY": "mcp-streamable-setup-rate-limit", + "LITELLM_RATE_LIMIT_WINDOW_SIZE": "30", + } + options: Final = { + "config": config, + "remove_environment": ("DATABASE_URL_READ_REPLICA", "LITELLM_LICENSE", "LITELLM_LICENSE_PATH"), + } + with owned_proxy(seed, tmp_path / "proxy", environment, **options) as gateway, gateway.scenario() as scenario: + server_id: Final = register_mcp(scenario, peer, "rpm", allowed_tools=["add"]) + key: Final = scenario.key(rpm_limit=2, object_permission={"mcp_servers": [server_id]}) + + def post_mcp(payload: dict[str, object], session_id: str | None = None) -> httpx.Response: + headers: Final = { + "Authorization": f"Bearer {key}", + "Accept": "application/json, text/event-stream", + **({"mcp-session-id": session_id} if session_id is not None else {}), + } + return gateway.client.post("/mcp/", headers=headers, json=payload) + + initialize: Final = post_mcp( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "litellm-test", "version": "1"}, + }, + } + ) + assert initialize.status_code == 200, initialize.text + session_id: Final = initialize.headers["mcp-session-id"] + + initialized: Final = post_mcp( + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + session_id, + ) + assert initialized.status_code == 202, initialized.text + + first_list: Final = post_mcp( + {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + session_id, + ) + assert first_list.status_code == 200, first_list.text + + second_list: Final = post_mcp( + {"jsonrpc": "2.0", "id": 3, "method": "tools/list", "params": {}}, + session_id, + ) + assert second_list.status_code == 429, second_list.text + assert second_list.headers.get("retry-after") is not None diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index de711d737d0..c0d8edf1a50 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2,6 +2,7 @@ import asyncio import json import os +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Final import pytest @@ -10,13 +11,14 @@ from contextlib import asynccontextmanager, nullcontext from starlette.types import Receive, Scope, Send - +from litellm.types.agents import AgentResponse from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, MCPServer, MCPTransport, ) from litellm.proxy._types import LiteLLM_ObjectPermissionTable +from litellm.proxy._types import UserAPIKeyAuth from mcp.types import Tool as MCPTool, CallToolResult, ListToolsResult from mcp.types import TextContent @@ -3259,6 +3261,267 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): assert await _read_request_body(_admission_request(scope)) == {} +@asynccontextmanager +async def _streamable_mcp_request_runner( + monkeypatch: pytest.MonkeyPatch, + auth: UserAPIKeyAuth, + on_receive_count_at_dispatch: Callable[[int], None] | None = None, +) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]: + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.utils import InternalUsageCache, ProxyLogging + + proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + limiter: Final = PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(dual_cache=DualCache()) + ) + proxy_logging.proxy_hook_mapping["parallel_request_limiter"] = limiter + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging) + + dispatch: Final = AsyncMock() + session_manager: Final = MagicMock() + session_manager.max_request_body_size = server.session_manager_stateless.max_request_body_size + session_manager.handle_request = dispatch + monkeypatch.setattr(server, "_SESSION_MANAGERS_INITIALIZED", True) + monkeypatch.setattr(server, "session_manager_stateless", session_manager) + monkeypatch.setattr(server, "session_manager_stateful", session_manager) + monkeypatch.setattr( + server, + "extract_mcp_auth_context", + AsyncMock(return_value=(auth, None, ["echo_server"], {}, {}, {})), + ) + + @asynccontextmanager + async def no_initialize_scope(*_args: object, **_kwargs: object) -> AsyncIterator[None]: + yield + + monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope) + + async def request( + method: str, + request_body: bytes | None = None, + request_chunks: tuple[bytes, ...] | None = None, + ) -> None: + body: Final = ( + request_body + if request_body is not None + else json.dumps( + { + "jsonrpc": "2.0", + "method": method, + **({"id": 1} if method != "notifications/initialized" else {}), + } + ).encode() + ) + scope: Final[Scope] = { + "type": "http", + "method": "POST", + "path": "/mcp/echo_server", + "headers": [(b"content-type", b"application/json")], + "query_string": b"", + "server": ("localhost", 4000), + "scheme": "http", + } + request_messages: Final = ( + tuple( + { + "type": "http.request", + "body": chunk, + "more_body": index < len(request_chunks) - 1, + } + for index, chunk in enumerate(request_chunks) + ) + if request_chunks is not None + else ({"type": "http.request", "body": body, "more_body": False},) + ) + receive: Final[Receive] = AsyncMock(side_effect=request_messages) + if on_receive_count_at_dispatch is not None: + def observe_receive_count(*_args: object, **_kwargs: object) -> None: + on_receive_count_at_dispatch(receive.await_count) + + dispatch.side_effect = observe_receive_count + send: Final[Send] = AsyncMock() + await server.handle_streamable_http_mcp(scope, receive, send) + + yield request, dispatch + + +def _padded_tools_list_body() -> bytes: + return json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + + +@pytest.mark.asyncio +async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth(api_key="mcp-setup-rpm-key", rpm_limit=2) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("initialize") + await request("tools/list") + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list") + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 2 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-setup-rpm-key", rpm_limit=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("tools/list") + oversized_body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth( + api_key="mcp-oversized-model-rpm-key", + metadata={"model_rpm_limit": {"mcp-tool-call": 1}}, + ) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = _padded_tools_list_body() + await request("tools/list", oversized_body) + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + agent_id: Final = "mcp-setup-agent-rpm" + agent_policy: Final = AgentResponse( + agent_id=agent_id, + agent_name="mcp-setup-agent", + agent_card_params={"name": "mcp-setup-agent"}, + rpm_limit=1, + ) + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy( + update={"managed_agent_policy": agent_policy} + ) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = _padded_tools_list_body() + await request("tools/list", oversized_body) + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-key") + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + await request("tools/list", oversized_body) + + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._experimental.mcp_server import server as mcp_server + + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key") + + def assert_receive_count(receive_count: int) -> None: + assert receive_count == 1 + + async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch): + assert await mcp_server._mcp_setup_request_limit_applies(auth) is False + oversized_body: Final = _padded_tools_list_body() + peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES + await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:])) + + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("tools/call") + await request("initialize") + + assert dispatch.await_count == 2 + + +@pytest.mark.asyncio +async def test_streamable_mcp_initialized_notification_does_not_consume_rpm( + monkeypatch: pytest.MonkeyPatch, +) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-initialized-rpm-key", rpm_limit=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("notifications/initialized") + await request("initialize") + + assert dispatch.await_count == 2 + + +@pytest.mark.asyncio +async def test_streamable_mcp_setup_request_releases_parallel_request_slot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-setup-parallel-key", max_parallel_requests=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("initialize") + await request("tools/list") + + assert dispatch.await_count == 2 + + @pytest.mark.asyncio async def test_mcp_routing_truncated_json_body_is_not_stashed_for_auth(): try: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index b8e28c3f9b8..81ad107b6b0 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -3006,7 +3006,7 @@ def test_oversized_initialize_peek_neither_routes_stateful_nor_attributes_client peeked = full_body[: mcp_server._MCP_ROUTING_PEEK_MAX_BYTES] assert mcp_server._extract_initialize_client_info(full_body) is not None - assert mcp_server._is_initialize_request(peeked) is False + assert mcp_server._mcp_request_method(peeked) is None assert mcp_server._extract_initialize_client_info(peeked) is None diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 1ec505362dd..4de06e42e43 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes( assert counter_scope in str(exc.value.detail) +@pytest.mark.asyncio +async def test_has_request_capacity_limit_matches_effective_request_descriptors(): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit")) + key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1) + model_rpm_auth = UserAPIKeyAuth( + api_key=hash_token("sk-count-model-request-limit"), + metadata={"model_rpm_limit": {"mcp-tool-call": 1}}, + ) + + assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False + assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True + assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True + + @pytest.mark.asyncio async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch): import litellm.proxy.proxy_server as proxy_server