mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
Merge f8bf45aae0 into 4710c3b30e
This commit is contained in:
commit
5342fdea34
6 changed files with 427 additions and 16 deletions
|
|
@ -115,6 +115,7 @@ if TYPE_CHECKING:
|
|||
|
||||
_redact_mcp_resource_url: Final = redact_mcp_resource_url
|
||||
|
||||
_MCP_SETUP_REQUEST_METHODS: Final = frozenset({"initialize", "tools/list"})
|
||||
_STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS: Final = 30 * 60
|
||||
# Upper bound on concurrent stateful sessions a single caller may hold. Each
|
||||
# `initialize` creates a session that survives until the idle timeout, so
|
||||
|
|
@ -135,6 +136,32 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version"
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _mcp_setup_request_capacity(
|
||||
method: str | None,
|
||||
user_api_key_auth: UserAPIKeyAuth | None,
|
||||
) -> contextlib.AbstractAsyncContextManager[None]:
|
||||
if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None:
|
||||
return contextlib.nullcontext()
|
||||
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
|
||||
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
|
||||
return contextlib.nullcontext()
|
||||
return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={})
|
||||
|
||||
|
||||
async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool:
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
|
||||
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
|
||||
return False
|
||||
return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call")
|
||||
|
||||
|
||||
def reject_disallowed_mcp_origin(request: StarletteRequest) -> None:
|
||||
from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup
|
||||
|
||||
|
|
@ -1403,18 +1430,15 @@ if MCP_AVAILABLE:
|
|||
return b"{}"
|
||||
return json.dumps({"method": method}).encode()
|
||||
|
||||
def _is_initialize_request(body: bytes) -> bool:
|
||||
"""
|
||||
Check if the request body is a JSON-RPC initialize method.
|
||||
Returns True if method is "initialize", False otherwise or on parse error.
|
||||
"""
|
||||
def _mcp_request_method(body: bytes) -> str | None:
|
||||
if not body:
|
||||
return False
|
||||
return None
|
||||
try:
|
||||
data: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(body)))
|
||||
return data is not None and data.get("method") == "initialize"
|
||||
method: Final = data.get("method") if data is not None else None
|
||||
return method if isinstance(method, str) else None
|
||||
except (json.JSONDecodeError, UnicodeDecodeError, TypeError):
|
||||
return False
|
||||
return None
|
||||
|
||||
def _jsonrpc_request_method(body: bytes) -> str | None:
|
||||
try:
|
||||
|
|
@ -2433,7 +2457,8 @@ if MCP_AVAILABLE:
|
|||
) = await extract_mcp_auth_context(scope, path)
|
||||
admit_mcp_request(scope, user_api_key_auth, mcp_servers)
|
||||
body: Final = await peek.body() if peek is not None else b""
|
||||
is_initialize: Final = _is_initialize_request(body)
|
||||
mcp_method: Final = _mcp_request_method(body)
|
||||
is_initialize: Final = mcp_method == "initialize"
|
||||
scoped_server_endpoint: Final = len(_get_mcp_servers_in_path(path) or []) == 1
|
||||
request_headers: Final = StarletteRequest(scope).headers
|
||||
defer_upstream_probes: Final = request_headers.get(
|
||||
|
|
@ -2715,11 +2740,20 @@ if MCP_AVAILABLE:
|
|||
if use_stateful and session_id and scope.get("method") == "DELETE":
|
||||
_remove_stateful_session_tracking(session_id)
|
||||
|
||||
charged_method: Final = (
|
||||
_jsonrpc_request_method(await peek.full_body())
|
||||
if mcp_method is None
|
||||
and peek is not None
|
||||
and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES
|
||||
and await _mcp_setup_request_limit_applies(user_api_key_auth)
|
||||
else mcp_method
|
||||
)
|
||||
request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth)
|
||||
try:
|
||||
if session_lock is not None:
|
||||
async with session_lock:
|
||||
await _dispatch()
|
||||
else:
|
||||
async with (
|
||||
session_lock if session_lock is not None else contextlib.nullcontext(),
|
||||
request_capacity,
|
||||
):
|
||||
await _dispatch()
|
||||
finally:
|
||||
for active_request_session_id in active_request_session_ids:
|
||||
|
|
|
|||
|
|
@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
*await self._create_tag_rate_limit_descriptors(data),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _request_capacity_data(
|
||||
model: str,
|
||||
request_data: Mapping[str, object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
return MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
|
||||
|
||||
async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool:
|
||||
data: Final = self._request_capacity_data(model)
|
||||
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
|
||||
return any(
|
||||
descriptor["rate_limit"] is not None
|
||||
and (
|
||||
descriptor["rate_limit"].get("requests_per_unit") is not None
|
||||
or descriptor["rate_limit"].get("max_parallel_requests") is not None
|
||||
)
|
||||
for descriptor in descriptors
|
||||
)
|
||||
|
||||
async def _release_request_capacity_when_admitted(
|
||||
self,
|
||||
admission: asyncio.Task[RateLimitResponse],
|
||||
|
|
@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
request_data: Mapping[str, object] | None = None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Charge one non-generation provider request to RPM and hold its concurrency slot."""
|
||||
data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
|
||||
data: Final = self._request_capacity_data(model, request_data)
|
||||
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
|
||||
acquisition: Final = ParallelSlotAcquisition(
|
||||
slot_id=uuid.uuid4().hex,
|
||||
|
|
|
|||
|
|
@ -269,3 +269,83 @@ def test_mcp_key_team_and_server_rpm_limits_share_redis(tmp_path: Path, monkeypa
|
|||
server_rejected: Final = _call_tool(second_replica, key_three, server_id, "rpm-add")
|
||||
_assert_rate_limit(server_rejected, "mcp_server")
|
||||
assert tool_calls(peer.drain()) == ()
|
||||
|
||||
|
||||
def test_streamable_mcp_setup_requests_use_virtual_key_rpm_limit(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
results_directory: Final = tmp_path / "results"
|
||||
results_directory.mkdir()
|
||||
monkeypatch.setenv("INTEGRATION_RESULTS_DIR", str(results_directory))
|
||||
assert os.environ.get("DATABASE_URL"), "This integration case requires disposable-database access"
|
||||
|
||||
with (
|
||||
owned_redis(tmp_path) as redis,
|
||||
scratch_database() as database_url,
|
||||
mcp_peer() as peer,
|
||||
httpx.Client() as client,
|
||||
):
|
||||
monkeypatch.setenv("DATABASE_URL", database_url)
|
||||
seed: Final = Gateway(client, "sk-mcp-streamable-setup-rate-limit", peer.url)
|
||||
config: Final = _config_file(
|
||||
tmp_path,
|
||||
seed.key,
|
||||
redis,
|
||||
upstream=peer.url,
|
||||
store_model_in_db=True,
|
||||
)
|
||||
environment: Final = {
|
||||
"DATABASE_URL": database_url,
|
||||
"LITELLM_SALT_KEY": "mcp-streamable-setup-rate-limit",
|
||||
"LITELLM_RATE_LIMIT_WINDOW_SIZE": "30",
|
||||
}
|
||||
options: Final = {
|
||||
"config": config,
|
||||
"remove_environment": ("DATABASE_URL_READ_REPLICA", "LITELLM_LICENSE", "LITELLM_LICENSE_PATH"),
|
||||
}
|
||||
with owned_proxy(seed, tmp_path / "proxy", environment, **options) as gateway, gateway.scenario() as scenario:
|
||||
server_id: Final = register_mcp(scenario, peer, "rpm", allowed_tools=["add"])
|
||||
key: Final = scenario.key(rpm_limit=2, object_permission={"mcp_servers": [server_id]})
|
||||
|
||||
def post_mcp(payload: dict[str, object], session_id: str | None = None) -> httpx.Response:
|
||||
headers: Final = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
**({"mcp-session-id": session_id} if session_id is not None else {}),
|
||||
}
|
||||
return gateway.client.post("/mcp/", headers=headers, json=payload)
|
||||
|
||||
initialize: Final = post_mcp(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2025-03-26",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "litellm-test", "version": "1"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert initialize.status_code == 200, initialize.text
|
||||
session_id: Final = initialize.headers["mcp-session-id"]
|
||||
|
||||
initialized: Final = post_mcp(
|
||||
{"jsonrpc": "2.0", "method": "notifications/initialized"},
|
||||
session_id,
|
||||
)
|
||||
assert initialized.status_code == 202, initialized.text
|
||||
|
||||
first_list: Final = post_mcp(
|
||||
{"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}},
|
||||
session_id,
|
||||
)
|
||||
assert first_list.status_code == 200, first_list.text
|
||||
|
||||
second_list: Final = post_mcp(
|
||||
{"jsonrpc": "2.0", "id": 3, "method": "tools/list", "params": {}},
|
||||
session_id,
|
||||
)
|
||||
assert second_list.status_code == 429, second_list.text
|
||||
assert second_list.headers.get("retry-after") is not None
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -10,13 +11,14 @@ from contextlib import asynccontextmanager, nullcontext
|
|||
|
||||
from starlette.types import Receive, Scope, Send
|
||||
|
||||
|
||||
from litellm.types.agents import AgentResponse
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
MCPServerManager,
|
||||
MCPServer,
|
||||
MCPTransport,
|
||||
)
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from mcp.types import Tool as MCPTool, CallToolResult, ListToolsResult
|
||||
from mcp.types import TextContent
|
||||
|
||||
|
|
@ -3259,6 +3261,267 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth():
|
|||
assert await _read_request_body(_admission_request(scope)) == {}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _streamable_mcp_request_runner(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
auth: UserAPIKeyAuth,
|
||||
on_receive_count_at_dispatch: Callable[[int], None] | None = None,
|
||||
) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._experimental.mcp_server import server
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
|
||||
from litellm.proxy.utils import InternalUsageCache, ProxyLogging
|
||||
|
||||
proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache())
|
||||
limiter: Final = PROXY_MaxParallelRequestsHandler_v3(
|
||||
internal_usage_cache=InternalUsageCache(dual_cache=DualCache())
|
||||
)
|
||||
proxy_logging.proxy_hook_mapping["parallel_request_limiter"] = limiter
|
||||
monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging)
|
||||
|
||||
dispatch: Final = AsyncMock()
|
||||
session_manager: Final = MagicMock()
|
||||
session_manager.max_request_body_size = server.session_manager_stateless.max_request_body_size
|
||||
session_manager.handle_request = dispatch
|
||||
monkeypatch.setattr(server, "_SESSION_MANAGERS_INITIALIZED", True)
|
||||
monkeypatch.setattr(server, "session_manager_stateless", session_manager)
|
||||
monkeypatch.setattr(server, "session_manager_stateful", session_manager)
|
||||
monkeypatch.setattr(
|
||||
server,
|
||||
"extract_mcp_auth_context",
|
||||
AsyncMock(return_value=(auth, None, ["echo_server"], {}, {}, {})),
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def no_initialize_scope(*_args: object, **_kwargs: object) -> AsyncIterator[None]:
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope)
|
||||
|
||||
async def request(
|
||||
method: str,
|
||||
request_body: bytes | None = None,
|
||||
request_chunks: tuple[bytes, ...] | None = None,
|
||||
) -> None:
|
||||
body: Final = (
|
||||
request_body
|
||||
if request_body is not None
|
||||
else json.dumps(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": method,
|
||||
**({"id": 1} if method != "notifications/initialized" else {}),
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
scope: Final[Scope] = {
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/mcp/echo_server",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"query_string": b"",
|
||||
"server": ("localhost", 4000),
|
||||
"scheme": "http",
|
||||
}
|
||||
request_messages: Final = (
|
||||
tuple(
|
||||
{
|
||||
"type": "http.request",
|
||||
"body": chunk,
|
||||
"more_body": index < len(request_chunks) - 1,
|
||||
}
|
||||
for index, chunk in enumerate(request_chunks)
|
||||
)
|
||||
if request_chunks is not None
|
||||
else ({"type": "http.request", "body": body, "more_body": False},)
|
||||
)
|
||||
receive: Final[Receive] = AsyncMock(side_effect=request_messages)
|
||||
if on_receive_count_at_dispatch is not None:
|
||||
def observe_receive_count(*_args: object, **_kwargs: object) -> None:
|
||||
on_receive_count_at_dispatch(receive.await_count)
|
||||
|
||||
dispatch.side_effect = observe_receive_count
|
||||
send: Final[Send] = AsyncMock()
|
||||
await server.handle_streamable_http_mcp(scope, receive, send)
|
||||
|
||||
yield request, dispatch
|
||||
|
||||
|
||||
def _padded_tools_list_body() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"params": {"_meta": {"pad": "x" * 5000}},
|
||||
"method": "tools/list",
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-setup-rpm-key", rpm_limit=2)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
await request("initialize")
|
||||
await request("tools/list")
|
||||
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
|
||||
await request("tools/list")
|
||||
|
||||
assert rate_limit_error.value.status_code == 429
|
||||
assert rate_limit_error.value.headers.get("retry-after") is not None
|
||||
assert dispatch.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-setup-rpm-key", rpm_limit=1)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
await request("tools/list")
|
||||
oversized_body: Final = json.dumps(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"params": {"_meta": {"pad": "x" * 5000}},
|
||||
"method": "tools/list",
|
||||
}
|
||||
).encode()
|
||||
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
|
||||
await request("tools/list", oversized_body)
|
||||
|
||||
assert rate_limit_error.value.status_code == 429
|
||||
assert rate_limit_error.value.headers.get("retry-after") is not None
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key="mcp-oversized-model-rpm-key",
|
||||
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
|
||||
)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
oversized_body: Final = _padded_tools_list_body()
|
||||
await request("tools/list", oversized_body)
|
||||
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
|
||||
await request("tools/list", oversized_body)
|
||||
|
||||
assert rate_limit_error.value.status_code == 429
|
||||
assert rate_limit_error.value.headers.get("retry-after") is not None
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
agent_id: Final = "mcp-setup-agent-rpm"
|
||||
agent_policy: Final = AgentResponse(
|
||||
agent_id=agent_id,
|
||||
agent_name="mcp-setup-agent",
|
||||
agent_card_params={"name": "mcp-setup-agent"},
|
||||
rpm_limit=1,
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy(
|
||||
update={"managed_agent_policy": agent_policy}
|
||||
)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
oversized_body: Final = _padded_tools_list_body()
|
||||
await request("tools/list", oversized_body)
|
||||
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
|
||||
await request("tools/list", oversized_body)
|
||||
|
||||
assert rate_limit_error.value.status_code == 429
|
||||
assert rate_limit_error.value.headers.get("retry-after") is not None
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-key")
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
oversized_body: Final = json.dumps(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"params": {"_meta": {"pad": "x" * 5000}},
|
||||
"method": "tools/list",
|
||||
}
|
||||
).encode()
|
||||
await request("tools/list", oversized_body)
|
||||
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy._experimental.mcp_server import server as mcp_server
|
||||
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key")
|
||||
|
||||
def assert_receive_count(receive_count: int) -> None:
|
||||
assert receive_count == 1
|
||||
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch):
|
||||
assert await mcp_server._mcp_setup_request_limit_applies(auth) is False
|
||||
oversized_body: Final = _padded_tools_list_body()
|
||||
peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES
|
||||
await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:]))
|
||||
|
||||
assert dispatch.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
await request("tools/call")
|
||||
await request("initialize")
|
||||
|
||||
assert dispatch.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_initialized_notification_does_not_consume_rpm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-initialized-rpm-key", rpm_limit=1)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
await request("notifications/initialized")
|
||||
await request("initialize")
|
||||
|
||||
assert dispatch.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_mcp_setup_request_releases_parallel_request_slot(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
auth: Final = UserAPIKeyAuth(api_key="mcp-setup-parallel-key", max_parallel_requests=1)
|
||||
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
|
||||
await request("initialize")
|
||||
await request("tools/list")
|
||||
|
||||
assert dispatch.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_routing_truncated_json_body_is_not_stashed_for_auth():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -3006,7 +3006,7 @@ def test_oversized_initialize_peek_neither_routes_stateful_nor_attributes_client
|
|||
peeked = full_body[: mcp_server._MCP_ROUTING_PEEK_MAX_BYTES]
|
||||
|
||||
assert mcp_server._extract_initialize_client_info(full_body) is not None
|
||||
assert mcp_server._is_initialize_request(peeked) is False
|
||||
assert mcp_server._mcp_request_method(peeked) is None
|
||||
assert mcp_server._extract_initialize_client_info(peeked) is None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes(
|
|||
assert counter_scope in str(exc.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_has_request_capacity_limit_matches_effective_request_descriptors():
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit"))
|
||||
key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1)
|
||||
model_rpm_auth = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-count-model-request-limit"),
|
||||
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
|
||||
)
|
||||
|
||||
assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False
|
||||
assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True
|
||||
assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue