mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(mcp): account for virtual tool calls once
This commit is contained in:
parent
dddbaa803d
commit
561d588f2b
4 changed files with 156 additions and 78 deletions
|
|
@ -321,39 +321,61 @@ async def _build_virtual_call_logging_obj(
|
|||
raw_headers: Mapping[str, str] | None = None,
|
||||
client_ip: str | None = None,
|
||||
) -> LiteLLMLoggingObj | None:
|
||||
"""Run the pre-call pipeline (guardrails + logging setup) for a virtual
|
||||
mcp_tool_call so the SSE path spend-logs like the REST path."""
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
)
|
||||
"""Build the virtual tool's logger without running request admission twice.
|
||||
|
||||
request: Final = build_synthetic_mcp_request(
|
||||
path="/mcp/tools/call",
|
||||
raw_headers=raw_headers,
|
||||
client_ip=client_ip,
|
||||
The real tool call owns admission and pre-call guardrails. Running the full
|
||||
proxy pre-call path here consumes a second RPM slot and parallel slot.
|
||||
"""
|
||||
from litellm.proxy.common_request_processing import resolve_litellm_call_id
|
||||
|
||||
headers: Final = dict(raw_headers) if raw_headers else {}
|
||||
call_id: Final = resolve_litellm_call_id(headers.get("x-litellm-call-id"))
|
||||
trace_id: Final = get_chain_id_from_headers(headers)
|
||||
request_data: dict[str, object] = {
|
||||
"model": f"MCP: {name}",
|
||||
"call_type": CallTypes.call_mcp_tool.value,
|
||||
"litellm_call_id": call_id,
|
||||
"litellm_trace_id": trace_id,
|
||||
"metadata": {
|
||||
"spend_logs_metadata": {
|
||||
"mcp_operation": "call_tool",
|
||||
"mcp_tool_name": name,
|
||||
},
|
||||
"headers": logging_safe_mcp_headers(raw_headers),
|
||||
**({"existing_trace_id": trace_id} if trace_id else {}),
|
||||
},
|
||||
"input": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": {
|
||||
"mcp_operation": "call_tool",
|
||||
"mcp_tool_name": name,
|
||||
"arguments": arguments,
|
||||
"client_ip": client_ip,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
auth_for_logging: Final = logging_auth or user_api_key_auth
|
||||
LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
|
||||
data=request_data,
|
||||
user_api_key_dict=auth_for_logging,
|
||||
_metadata_variable_name="metadata",
|
||||
)
|
||||
pre_call_result: Final = await ProxyBaseLLMRequestProcessing(
|
||||
data={"name": name, "arguments": arguments}
|
||||
).common_processing_pre_call_logic(
|
||||
request=request,
|
||||
user_api_key_dict=user_api_key_auth,
|
||||
proxy_config=proxy_config,
|
||||
route_type=CallTypes.call_mcp_tool.value,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings=general_settings,
|
||||
logging_auth=logging_auth,
|
||||
)
|
||||
_, logging_candidate = TypeAdapter(tuple[dict[str, object], object]).validate_python(pre_call_result)
|
||||
if not isinstance(logging_candidate, LiteLLMLoggingObj):
|
||||
verbose_logger.warning("MCP request processing did not return a logging object")
|
||||
try:
|
||||
logging_obj, _ = function_setup(
|
||||
original_function=CallTypes.call_mcp_tool.value,
|
||||
is_async_call=True,
|
||||
rules_obj=Rules(),
|
||||
start_time=datetime.now(), # noqa: DTZ005 # logging timestamps are naive
|
||||
**request_data,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 # logging setup must not block the tool call
|
||||
verbose_logger.debug("Failed to initialize logging for MCP virtual tool %s: %s", name, exc)
|
||||
return None
|
||||
virtual_logging_obj: Final = logging_candidate
|
||||
return virtual_logging_obj
|
||||
logging_obj.call_type = CallTypes.call_mcp_tool.value
|
||||
logging_obj.model = f"MCP: {name}"
|
||||
return logging_obj
|
||||
|
||||
|
||||
async def _log_virtual_mcp_tool_failure(
|
||||
|
|
@ -595,6 +617,7 @@ async def _dispatch_virtual_mcp_tool(
|
|||
tool_name=tool_request.name,
|
||||
mcp_servers=mcp_servers,
|
||||
)
|
||||
virtual_call_start: Final = datetime.now() # noqa: DTZ005 # logging timestamps are naive
|
||||
virtual_logging_obj: Final = await _build_virtual_call_logging_obj(
|
||||
name=tool_request.name,
|
||||
arguments=tool_request.arguments or {},
|
||||
|
|
@ -603,19 +626,45 @@ async def _dispatch_virtual_mcp_tool(
|
|||
raw_headers=raw_headers,
|
||||
client_ip=client_ip,
|
||||
)
|
||||
return await handle_mcp_tool_call(
|
||||
tool_name=tool_request.name,
|
||||
arguments=tool_request.arguments or {},
|
||||
user_api_key_dict=user_api_key_auth,
|
||||
client_ip=client_ip,
|
||||
mcp_servers=mcp_servers,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
litellm_logging_obj=virtual_logging_obj,
|
||||
incoming_bearer_token=incoming_bearer_token,
|
||||
guardrail_auth=tool_guardrail_auth,
|
||||
request_data: Final = types.MappingProxyType(
|
||||
{
|
||||
"name": tool_request.name,
|
||||
"arguments": tool_request.arguments or {},
|
||||
}
|
||||
)
|
||||
try:
|
||||
result: Final = await handle_mcp_tool_call(
|
||||
tool_name=tool_request.name,
|
||||
arguments=tool_request.arguments or {},
|
||||
user_api_key_dict=user_api_key_auth,
|
||||
client_ip=client_ip,
|
||||
mcp_servers=mcp_servers,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
litellm_logging_obj=virtual_logging_obj,
|
||||
incoming_bearer_token=incoming_bearer_token,
|
||||
guardrail_auth=tool_guardrail_auth,
|
||||
)
|
||||
except Exception as exc:
|
||||
await fire_mcp_tool_call_failure_logging(
|
||||
virtual_logging_obj,
|
||||
exc,
|
||||
virtual_call_start,
|
||||
user_api_key_auth,
|
||||
request_data,
|
||||
)
|
||||
raise
|
||||
if virtual_logging_obj is None:
|
||||
return result
|
||||
return await _fire_mcp_tool_call_logging(
|
||||
logging_obj=virtual_logging_obj,
|
||||
result=result,
|
||||
start_time=virtual_call_start,
|
||||
end_time=datetime.now(), # noqa: DTZ005 # logging timestamps are naive
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -247,6 +247,7 @@ if MCP_AVAILABLE:
|
|||
MCPInfo,
|
||||
MCPServer,
|
||||
_aggregate_server_key,
|
||||
_build_virtual_call_logging_obj,
|
||||
_fire_mcp_tool_call_logging,
|
||||
execute_mcp_tool,
|
||||
filter_tools_by_allowed_tools,
|
||||
|
|
@ -348,8 +349,6 @@ if MCP_AVAILABLE:
|
|||
handle_mcp_tool_search,
|
||||
handle_skill_search,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.proxy_server import general_settings, proxy_config, proxy_logging_obj
|
||||
|
||||
if not getattr(getattr(user_api_key_dict, "object_permission", None), "mcp_tool_search_enabled", False):
|
||||
raise HTTPException(
|
||||
|
|
@ -391,20 +390,28 @@ if MCP_AVAILABLE:
|
|||
oauth2_headers=virtual_oauth2_headers,
|
||||
raw_headers=virtual_raw_headers,
|
||||
)
|
||||
# MCP_TOOL_CALL_TOOL_NAME: run the same pre-call pipeline as the normal path so the tool
|
||||
# execution is spend-logged and guardrail-checked.
|
||||
virtual_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||
# MCP_TOOL_CALL_TOOL_NAME: the real tool call below owns admission and guardrails.
|
||||
# Building the logger separately prevents a second RPM or parallel slot charge.
|
||||
_request_start_time: Final = datetime.now() # noqa: DTZ005 # naive to match the tool start time below
|
||||
try:
|
||||
(virtual_data, virtual_logging_obj) = await virtual_processor.common_processing_pre_call_logic(
|
||||
request=request,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_config=proxy_config,
|
||||
route_type=CallTypes.call_mcp_tool.value,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings=general_settings,
|
||||
virtual_logging_obj: Final = await _build_virtual_call_logging_obj(
|
||||
name=tool_arguments.get("tool_name", ""),
|
||||
arguments=tool_arguments.get("arguments") or {},
|
||||
user_api_key_auth=user_api_key_dict,
|
||||
raw_headers=virtual_raw_headers,
|
||||
client_ip=rest_client_ip,
|
||||
)
|
||||
_tool_start_time: Final = datetime.now()
|
||||
except Exception as e:
|
||||
await _safe_fire_mcp_tool_call_failure_logging(
|
||||
None,
|
||||
e,
|
||||
_request_start_time,
|
||||
user_api_key_dict,
|
||||
data,
|
||||
)
|
||||
raise
|
||||
_tool_start_time: Final = datetime.now()
|
||||
try:
|
||||
result: Final = await handle_mcp_tool_call(
|
||||
tool_name=tool_arguments.get("tool_name", ""),
|
||||
arguments=tool_arguments.get("arguments") or {},
|
||||
|
|
@ -415,17 +422,16 @@ if MCP_AVAILABLE:
|
|||
oauth2_headers=virtual_oauth2_headers,
|
||||
raw_headers=virtual_raw_headers,
|
||||
litellm_logging_obj=virtual_logging_obj,
|
||||
guardrail_context=MCPRequestContext.resolve_guardrail_context(virtual_data),
|
||||
guardrail_context=MCPRequestContext.resolve_guardrail_context(data),
|
||||
incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token(request.headers, user_api_key_dict),
|
||||
)
|
||||
except Exception as e:
|
||||
virtual_request_data: Final = virtual_processor.data
|
||||
await _safe_fire_mcp_tool_call_failure_logging(
|
||||
virtual_request_data.get("litellm_logging_obj"),
|
||||
virtual_logging_obj,
|
||||
e,
|
||||
_request_start_time,
|
||||
user_api_key_dict,
|
||||
virtual_request_data,
|
||||
data,
|
||||
)
|
||||
raise
|
||||
return await _safe_fire_mcp_tool_call_logging(
|
||||
|
|
|
|||
|
|
@ -1144,16 +1144,14 @@ class TestDispatchVirtualMcpTool:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_builds_and_forwards_logging_obj(self) -> None:
|
||||
"""Regression: the SSE dispatch must run the pre-call pipeline and forward
|
||||
the resulting logging object to handle_mcp_tool_call, otherwise mcp_tool_call
|
||||
over /mcp/ skips spend logging and guardrails (unlike the REST path)."""
|
||||
from litellm.proxy._experimental.mcp_server import server as srv
|
||||
"""The virtual call forwards one logger and logs after the real tool call."""
|
||||
|
||||
uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
|
||||
sentinel_logging_obj = object()
|
||||
with (
|
||||
patch.object(
|
||||
mcp_operations, "_build_virtual_call_logging_obj",
|
||||
mcp_operations,
|
||||
"_build_virtual_call_logging_obj",
|
||||
new_callable=AsyncMock,
|
||||
return_value=sentinel_logging_obj,
|
||||
) as mock_build,
|
||||
|
|
@ -1162,8 +1160,14 @@ class TestDispatchVirtualMcpTool:
|
|||
new_callable=AsyncMock,
|
||||
return_value="CALL_RESULT",
|
||||
) as mock_call,
|
||||
patch.object(
|
||||
mcp_operations,
|
||||
"_fire_mcp_tool_call_logging",
|
||||
new_callable=AsyncMock,
|
||||
return_value="LOGGED_RESULT",
|
||||
) as mock_fire,
|
||||
):
|
||||
await mcp_operations._dispatch_virtual_mcp_tool(
|
||||
result = await mcp_operations._dispatch_virtual_mcp_tool(
|
||||
name=MCP_TOOL_CALL_TOOL_NAME,
|
||||
arguments={"tool_name": "math-add", "arguments": {"a": 1}},
|
||||
user_api_key_auth=uak,
|
||||
|
|
@ -1172,6 +1176,26 @@ class TestDispatchVirtualMcpTool:
|
|||
|
||||
assert mock_build.await_count == 1
|
||||
assert mock_call.await_args.kwargs["litellm_logging_obj"] is sentinel_logging_obj
|
||||
assert mock_fire.await_count == 1
|
||||
assert result == "LOGGED_RESULT"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_virtual_logger_does_not_consume_request_admission(self, monkeypatch) -> None:
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
pre_call_hook = AsyncMock()
|
||||
monkeypatch.setattr(proxy_logging_obj, "pre_call_hook", pre_call_hook)
|
||||
uak = UserAPIKeyAuth(api_key="k", user_id="user-1")
|
||||
|
||||
logging_obj = await mcp_operations._build_virtual_call_logging_obj(
|
||||
name="math-add",
|
||||
arguments={"a": 1},
|
||||
user_api_key_auth=uak,
|
||||
)
|
||||
|
||||
assert logging_obj is not None
|
||||
assert logging_obj.model == "MCP: math-add"
|
||||
pre_call_hook.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_coerces_non_int_top_k(self) -> None:
|
||||
|
|
|
|||
|
|
@ -3524,7 +3524,6 @@ class TestCallToolRestAPI:
|
|||
async def test_virtual_mcp_tool_call_guardrail_block_runs_failure_logging(self, monkeypatch):
|
||||
"""A pre_mcp_call guardrail block on the virtual mcp_tool_call branch must write a failure
|
||||
spend log, same as the direct tool call branch, and still raise the original error."""
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
|
||||
guardrail_error = HTTPException(status_code=400, detail={"error": "Content blocked"})
|
||||
|
|
@ -3532,20 +3531,20 @@ class TestCallToolRestAPI:
|
|||
async def fake_contexts(user_api_key_auth):
|
||||
return [user_api_key_auth]
|
||||
|
||||
async def fake_add_litellm_data_to_request(**kwargs):
|
||||
return kwargs.get("data", {})
|
||||
|
||||
async def blocking_pre_call_hook(user_api_key_dict, data, call_type, skip_guardrails=False):
|
||||
raise guardrail_error
|
||||
|
||||
failure_logging = AsyncMock()
|
||||
monkeypatch.setattr(rest_endpoints, "build_effective_auth_contexts", fake_contexts, raising=False)
|
||||
monkeypatch.setattr(
|
||||
proxy_server, "add_litellm_data_to_request", fake_add_litellm_data_to_request, raising=False
|
||||
mcp_operations,
|
||||
"_get_allowed_mcp_servers",
|
||||
AsyncMock(return_value=[MagicMock()]),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mcp_operations,
|
||||
"execute_mcp_tool",
|
||||
AsyncMock(side_effect=guardrail_error),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", {}, raising=False)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False)
|
||||
monkeypatch.setattr(proxy_server.proxy_logging_obj, "pre_call_hook", blocking_pre_call_hook)
|
||||
monkeypatch.setattr(rest_endpoints, "fire_mcp_tool_call_failure_logging", failure_logging, raising=False)
|
||||
|
||||
user_api_key_dict = UserAPIKeyAuth(
|
||||
|
|
@ -3570,8 +3569,8 @@ class TestCallToolRestAPI:
|
|||
logging_obj, exception, _start_time, user_api_key_auth, request_data = failure_logging.await_args.args
|
||||
assert exception is guardrail_error
|
||||
assert user_api_key_auth is user_api_key_dict
|
||||
assert logging_obj is request_data.get("litellm_logging_obj")
|
||||
assert logging_obj is not None
|
||||
assert request_data["name"] == "mcp_tool_call"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue