mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 04af109ee1 into 4573fa1f31
This commit is contained in:
commit
f515b9b1aa
12 changed files with 764 additions and 19 deletions
62
litellm/litellm_core_utils/mcp_tool_replay.py
Normal file
62
litellm/litellm_core_utils/mcp_tool_replay.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
from collections.abc import Iterator
|
||||
from types import TracebackType
|
||||
from typing import Final
|
||||
|
||||
_MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed"
|
||||
|
||||
_MAX_EXCEPTION_CHAIN_DEPTH: Final = 20
|
||||
|
||||
|
||||
def mark_mcp_tools_executed(exception: BaseException) -> None:
|
||||
"""Tag an error raised after an MCP gateway loop already executed tool calls.
|
||||
|
||||
Retrying or falling back re-runs the whole loop, which executes those tool calls
|
||||
again, so the router surfaces a tagged error instead of replaying the request. The
|
||||
exception is tagged rather than wrapped so the client sees the same error.
|
||||
"""
|
||||
setattr(exception, _MCP_TOOLS_EXECUTED_ATTR, True)
|
||||
|
||||
|
||||
def _exception_chain(exception: BaseException) -> Iterator[BaseException]:
|
||||
current = exception # rebind-ok: advances one link per iteration of the bounded walk
|
||||
for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
|
||||
yield current
|
||||
following = current.__cause__ or current.__context__
|
||||
if following is None:
|
||||
return
|
||||
current = following
|
||||
|
||||
|
||||
def mcp_tools_executed(exception: BaseException | None) -> bool:
|
||||
"""Whether ``exception`` or an exception in its chain was tagged by ``mark_mcp_tools_executed``.
|
||||
|
||||
The chain matters because a caller can re-raise a tagged error as a new exception, as
|
||||
``exception_type`` does for anything that is not already a LiteLLM exception.
|
||||
"""
|
||||
return exception is not None and any(
|
||||
getattr(link, _MCP_TOOLS_EXECUTED_ATTR, False) is True for link in _exception_chain(exception)
|
||||
)
|
||||
|
||||
|
||||
class MCPToolReplayGuard:
|
||||
"""Tags errors raised inside ``with guard:`` once the request has sent an MCP tool call to a server."""
|
||||
|
||||
__slots__ = ("_tool_dispatched",)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tool_dispatched: bool = False
|
||||
|
||||
def record_tool_dispatch(self) -> None:
|
||||
self._tool_dispatched = True
|
||||
|
||||
def __enter__(self) -> None:
|
||||
return None
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
if self._tool_dispatched and isinstance(exc, Exception):
|
||||
mark_mcp_tools_executed(exc)
|
||||
|
|
@ -12,6 +12,7 @@ from types import MappingProxyType
|
|||
from typing import Any, Final, NamedTuple
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.mcp_tool_replay import MCPToolReplayGuard
|
||||
from litellm.responses.mcp.request_context import MCPRequestContext
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicMessagesTool,
|
||||
|
|
@ -149,6 +150,7 @@ async def anthropic_messages_with_mcp(
|
|||
messages=list(working_messages), stream=False, **loop_call_args
|
||||
)
|
||||
|
||||
replay_guard: Final = MCPToolReplayGuard()
|
||||
for _ in range(MAX_MCP_TOOL_USE_ITERATIONS):
|
||||
if _get_stop_reason(response) != "tool_use":
|
||||
break
|
||||
|
|
@ -170,6 +172,7 @@ async def anthropic_messages_with_mcp(
|
|||
litellm_trace_id=context.litellm_trace_id,
|
||||
request_tags=list(context.request_tags) if context.request_tags else None,
|
||||
guardrail_context=context.guardrail_context,
|
||||
on_tool_dispatched=replay_guard.record_tool_dispatch,
|
||||
)
|
||||
|
||||
# Every tool call was skipped, so there is nothing to feed back; a
|
||||
|
|
@ -182,9 +185,10 @@ async def anthropic_messages_with_mcp(
|
|||
{"role": "assistant", "content": list(_get_response_content(response))},
|
||||
_build_tool_result_message(tool_results),
|
||||
)
|
||||
response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn(
|
||||
messages=list(working_messages), stream=False, **loop_call_args
|
||||
)
|
||||
with replay_guard:
|
||||
response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn(
|
||||
messages=list(working_messages), stream=False, **loop_call_args
|
||||
)
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
"MCP tool loop hit its %s iteration cap for model %s; returning the last response",
|
||||
|
|
@ -192,11 +196,12 @@ async def anthropic_messages_with_mcp(
|
|||
model,
|
||||
)
|
||||
|
||||
client_response: Final = (
|
||||
await with_safeguard_results(response, request_safeguards_evaluator(kwargs, working_messages))
|
||||
if classified_here
|
||||
else response
|
||||
)
|
||||
with replay_guard:
|
||||
client_response: Final = (
|
||||
await with_safeguard_results(response, request_safeguards_evaluator(kwargs, working_messages))
|
||||
if classified_here
|
||||
else response
|
||||
)
|
||||
if stream:
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
|
|
|
|||
|
|
@ -1146,6 +1146,7 @@ class MCPClient:
|
|||
raise_on_error: bool = False,
|
||||
allow_input_required: bool = False,
|
||||
persistent_session: "PersistentMCPSession | None" = None,
|
||||
on_dispatch: Callable[[], None] | None = None,
|
||||
) -> MCPCallToolResult | InputRequiredResult:
|
||||
"""
|
||||
Call an MCP Tool.
|
||||
|
|
@ -1162,6 +1163,8 @@ class MCPClient:
|
|||
allow_input_required: When True, a 2026-07-28 upstream may answer with an interim
|
||||
``InputRequiredResult`` and it is returned as is. The SDK rejects it otherwise, so
|
||||
callers only opt in when the downstream side can carry it.
|
||||
on_dispatch: Called right before the tool call is sent on a ready session, so a caller
|
||||
can tell a call the server may have run from one that never left.
|
||||
"""
|
||||
verbose_logger.info("MCP client calling tool '%s'", call_tool_request_params.name)
|
||||
|
||||
|
|
@ -1195,6 +1198,8 @@ class MCPClient:
|
|||
async def request(
|
||||
responses: InputResponses | None, state: str | None
|
||||
) -> MCPCallToolResult | InputRequiredResult:
|
||||
if on_dispatch is not None:
|
||||
on_dispatch()
|
||||
return await session.call_tool(
|
||||
name=call_tool_request_params.name,
|
||||
arguments=call_tool_request_params.arguments,
|
||||
|
|
|
|||
|
|
@ -6540,6 +6540,7 @@ class MCPServerManager:
|
|||
client_ip: str | None = None,
|
||||
allow_input_required: bool = False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch: Callable[[], None] | None = None,
|
||||
) -> CallToolResult | InputRequiredResult:
|
||||
"""Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry.
|
||||
|
||||
|
|
@ -6556,6 +6557,7 @@ class MCPServerManager:
|
|||
raise_on_error=True,
|
||||
allow_input_required=allow_input_required,
|
||||
persistent_session=persistent_session,
|
||||
on_dispatch=on_dispatch,
|
||||
)
|
||||
except Exception as exc:
|
||||
auth_failure: Final = _extract_upstream_auth_failure(exc)
|
||||
|
|
@ -6583,6 +6585,7 @@ class MCPServerManager:
|
|||
host_progress_callback=host_progress_callback,
|
||||
allow_input_required=allow_input_required,
|
||||
persistent_session=retry_persistent_session,
|
||||
on_dispatch=on_dispatch,
|
||||
)
|
||||
|
||||
async def _call_regular_mcp_tool(
|
||||
|
|
@ -6603,6 +6606,7 @@ class MCPServerManager:
|
|||
allow_input_required: bool = False,
|
||||
input_responses: InputResponses | None = None,
|
||||
request_state: str | None = None,
|
||||
on_dispatch: Callable[[], None] | None = None,
|
||||
) -> CallToolResult | InputRequiredResult:
|
||||
"""
|
||||
Call a regular MCP tool using the MCP client.
|
||||
|
|
@ -6759,6 +6763,7 @@ class MCPServerManager:
|
|||
client_ip=client_ip,
|
||||
allow_input_required=allow_input_required,
|
||||
persistent_session=persistent_session,
|
||||
on_dispatch=on_dispatch,
|
||||
)
|
||||
else:
|
||||
# Scoped to the two client-forwarded token modes this stack introduced; legacy
|
||||
|
|
@ -6776,6 +6781,7 @@ class MCPServerManager:
|
|||
host_progress_callback=host_progress_callback,
|
||||
allow_input_required=allow_input_required,
|
||||
persistent_session=persistent_session,
|
||||
on_dispatch=on_dispatch,
|
||||
)
|
||||
# The client-forwarded modes carry the caller's own upstream token, so an upstream
|
||||
# 401 (expired/invalid token) is the caller's to resolve: relay it as
|
||||
|
|
@ -6793,6 +6799,7 @@ class MCPServerManager:
|
|||
raise_on_error=True,
|
||||
allow_input_required=allow_input_required,
|
||||
persistent_session=persistent_session,
|
||||
on_dispatch=on_dispatch,
|
||||
)
|
||||
except Exception as e:
|
||||
required_scope: Final = upstream_insufficient_scope(e) if managed_per_user else None
|
||||
|
|
@ -7072,6 +7079,7 @@ class MCPServerManager:
|
|||
wire_compat: WireCompat = WireCompat.LEGACY,
|
||||
input_responses: InputResponses | None = None,
|
||||
request_state: str | None = None,
|
||||
on_dispatch: Callable[[], None] | None = None,
|
||||
*,
|
||||
guardrail_auth: UserAPIKeyAuth | None = None,
|
||||
catalog_auth_header: str | None | EllipsisType = ...,
|
||||
|
|
@ -7094,6 +7102,8 @@ class MCPServerManager:
|
|||
litellm_logging_obj: Optional request logger the guardrail hooks record
|
||||
their evaluations onto, so MCP guardrail activity reaches the
|
||||
Guardrails Monitor. See ``pre_call_tool_check``
|
||||
on_dispatch: Optional callback run right before the call is sent to the server,
|
||||
after its headers and client are ready, even if the server never returns a result
|
||||
|
||||
|
||||
Returns:
|
||||
|
|
@ -7214,6 +7224,8 @@ class MCPServerManager:
|
|||
resolved_token: Final = request_resolved_auth_headers.set(resolved_auth_headers)
|
||||
try:
|
||||
async with self._limit_outbound_concurrency(mcp_server):
|
||||
if on_dispatch is not None:
|
||||
on_dispatch()
|
||||
return await self._call_openapi_tool_handler(mcp_server, name, arguments, wire_compat)
|
||||
finally:
|
||||
request_auth_header.reset(auth_token)
|
||||
|
|
@ -7239,6 +7251,7 @@ class MCPServerManager:
|
|||
allow_input_required=wire_compat is WireCompat.MODERN,
|
||||
input_responses=input_responses,
|
||||
request_state=request_state,
|
||||
on_dispatch=on_dispatch,
|
||||
)
|
||||
|
||||
return await self._gather_openapi_tool_tasks(tasks, proxy_logging_obj)
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.mcp_tool_replay import MCPToolReplayGuard
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
update_responses_input_with_model_file_ids,
|
||||
update_responses_tools_with_model_file_ids,
|
||||
|
|
@ -472,6 +473,7 @@ async def aresponses_api_with_mcp(
|
|||
)
|
||||
|
||||
tool_trace_id: Final = _request_trace_id(request_kwargs)
|
||||
replay_guard: Final = MCPToolReplayGuard()
|
||||
tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls(
|
||||
tool_server_map=tool_server_map,
|
||||
served_tools=original_mcp_tools,
|
||||
|
|
@ -488,6 +490,7 @@ async def aresponses_api_with_mcp(
|
|||
MappingProxyType({**kwargs, "metadata": metadata, "model": model})
|
||||
),
|
||||
request_trace_id=_langfuse_request_trace_id(request_kwargs, tool_trace_id),
|
||||
on_tool_dispatched=replay_guard.record_tool_dispatch,
|
||||
)
|
||||
|
||||
if tool_results:
|
||||
|
|
@ -514,8 +517,11 @@ async def aresponses_api_with_mcp(
|
|||
tool_calls=tool_calls, tool_results=tool_results
|
||||
)
|
||||
|
||||
with internal_sub_call(
|
||||
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
|
||||
with (
|
||||
internal_sub_call(
|
||||
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
|
||||
),
|
||||
replay_guard,
|
||||
):
|
||||
final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call(
|
||||
follow_up_input=follow_up_input,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing_extensions import TypedDict, Unpack
|
|||
from litellm._internal_context import internal_sub_call
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.mcp_tool_replay import MCPToolReplayGuard
|
||||
from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
|
|
@ -261,6 +262,7 @@ async def acompletion_with_mcp(
|
|||
self.follow_up_stream = None
|
||||
self.follow_up_iterator = None
|
||||
self.follow_up_exhausted = False
|
||||
self.replay_guard = MCPToolReplayGuard()
|
||||
self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj)
|
||||
self._unsent_usage_chunk: ModelResponseStream | None = None
|
||||
|
||||
|
|
@ -354,7 +356,8 @@ async def acompletion_with_mcp(
|
|||
|
||||
async def _follow_up_or_release_first_round(self, usage_chunk: ModelResponseStream | None) -> None:
|
||||
if self.tool_results and self.complete_response:
|
||||
await self._prepare_follow_up_call()
|
||||
with self.replay_guard:
|
||||
await self._prepare_follow_up_call()
|
||||
if self.follow_up_stream is not None:
|
||||
return
|
||||
self._unsent_usage_chunk = usage_chunk
|
||||
|
|
@ -438,7 +441,8 @@ async def acompletion_with_mcp(
|
|||
verbose_logger.debug("Follow-up stream iterator created")
|
||||
|
||||
try:
|
||||
chunk = await self.follow_up_iterator.__anext__()
|
||||
with self.replay_guard:
|
||||
chunk = await self.follow_up_iterator.__anext__()
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.debug("Follow-up chunk yielded: %s", chunk)
|
||||
|
|
@ -509,6 +513,7 @@ async def acompletion_with_mcp(
|
|||
litellm_trace_id=self.litellm_trace_id,
|
||||
request_tags=self.request_tags,
|
||||
guardrail_context=context.guardrail_context,
|
||||
on_tool_dispatched=self.replay_guard.record_tool_dispatch,
|
||||
)
|
||||
|
||||
async def _prepare_follow_up_call(self):
|
||||
|
|
@ -679,6 +684,7 @@ async def acompletion_with_mcp(
|
|||
)
|
||||
return initial_response
|
||||
|
||||
replay_guard: Final = MCPToolReplayGuard()
|
||||
with first_round_billed_on_failure(logging_obj, initial_response):
|
||||
tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls(
|
||||
tool_server_map=tool_server_map,
|
||||
|
|
@ -693,6 +699,7 @@ async def acompletion_with_mcp(
|
|||
litellm_trace_id=context.litellm_trace_id,
|
||||
request_tags=request_tags,
|
||||
guardrail_context=context.guardrail_context,
|
||||
on_tool_dispatched=replay_guard.record_tool_dispatch,
|
||||
)
|
||||
|
||||
if not tool_results:
|
||||
|
|
@ -713,7 +720,10 @@ async def acompletion_with_mcp(
|
|||
follow_up_call_args["messages"] = follow_up_messages
|
||||
follow_up_call_args["stream"] = stream
|
||||
|
||||
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
|
||||
with (
|
||||
internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None),
|
||||
replay_guard,
|
||||
):
|
||||
final_response: Final = await litellm_acompletion(**follow_up_call_args)
|
||||
|
||||
if not isinstance(final_response, ModelResponse):
|
||||
|
|
|
|||
|
|
@ -740,8 +740,14 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
guardrail_context: Mapping[str, object] | None = None,
|
||||
served_tools: Sequence[MCPTool] | None = None,
|
||||
request_trace_id: str | None = None,
|
||||
on_tool_dispatched: Callable[[], None] | None = None,
|
||||
) -> list[MCPToolResult]:
|
||||
"""Execute tool calls and return results."""
|
||||
"""Execute tool calls and return results.
|
||||
|
||||
``on_tool_dispatched`` runs for every call that is sent to its MCP server, even one that then
|
||||
times out. Calls rejected or failing before that, for example on a pre-call guardrail or a
|
||||
credential the caller has not set, do not trigger it.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -908,6 +914,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
guardrail_context=guardrail_context,
|
||||
on_dispatch=on_tool_dispatched,
|
||||
listed_tool=(
|
||||
next((tool for tool in served_tools if tool.name == tool_name), None)
|
||||
if served_tools is not None
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
|
||||
from litellm.litellm_core_utils.mcp_tool_replay import mcp_tools_executed
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
PTU_COST_ATTRIBUTION_ENV_VAR,
|
||||
|
|
@ -9135,6 +9136,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
or original_model_group is None
|
||||
or is_guardrail_intervention(e)
|
||||
or unsupported_params_without_fallback
|
||||
or mcp_tools_executed(e)
|
||||
):
|
||||
raise e
|
||||
|
||||
|
|
@ -9565,7 +9567,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None)
|
||||
return response
|
||||
except Exception as e:
|
||||
if is_guardrail_intervention(e):
|
||||
if is_guardrail_intervention(e) or mcp_tools_executed(e):
|
||||
raise
|
||||
current_attempt = None
|
||||
original_exception = e
|
||||
|
|
@ -9676,6 +9678,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# Always track the latest error so we raise the most
|
||||
# recent exception instead of the first one.
|
||||
original_exception = e
|
||||
if mcp_tools_executed(e):
|
||||
raise
|
||||
|
||||
## LOGGING
|
||||
kwargs = self._core_log_retry(kwargs=kwargs, e=e)
|
||||
|
|
|
|||
|
|
@ -1035,6 +1035,36 @@ async def test_call_tool_does_not_log_arguments():
|
|||
assert "gpt-5-mini" not in logged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("session_opens", [True, False])
|
||||
async def test_call_tool_reports_dispatch_only_when_the_call_is_sent(session_opens):
|
||||
"""
|
||||
on_dispatch marks the moment a tool call can reach the server, so it fires right before the
|
||||
session sends the call and never when the session could not be opened.
|
||||
"""
|
||||
from mcp.types import CallToolRequestParams
|
||||
|
||||
events: list[str] = []
|
||||
|
||||
async def send(**_):
|
||||
events.append("sent")
|
||||
return MagicMock()
|
||||
|
||||
async def run_with_session(operation, quiet_on_error=False):
|
||||
if not session_opens:
|
||||
raise ConnectionError("cannot reach the server")
|
||||
return await operation(MagicMock(call_tool=send))
|
||||
|
||||
client = MCPClient(server_url="http://test-server")
|
||||
client.run_with_session = run_with_session
|
||||
|
||||
await client.call_tool(
|
||||
CallToolRequestParams(name="search_tool", arguments={}), on_dispatch=lambda: events.append("dispatched")
|
||||
)
|
||||
|
||||
assert events == (["dispatched", "sent"] if session_opens else [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_prompt_does_not_log_arguments():
|
||||
from mcp.types import GetPromptRequestParams
|
||||
|
|
|
|||
|
|
@ -67,8 +67,9 @@ class _AdmissionProbeClient(MCPClient):
|
|||
raise_on_error: bool = False,
|
||||
allow_input_required: bool = False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch: Callable[[], None] | None = None,
|
||||
) -> CallToolResult:
|
||||
del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required
|
||||
del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required, on_dispatch
|
||||
if self._submitted is not None:
|
||||
self._submitted.set()
|
||||
if persistent_session is None:
|
||||
|
|
@ -121,6 +122,7 @@ def _patch_client_with_tracker(manager: MCPServerManager, tracker: _ConcurrencyT
|
|||
host_progress_callback=None,
|
||||
allow_input_required=False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch=None,
|
||||
):
|
||||
tracker.enter(server.server_id)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -7644,6 +7644,7 @@ class TestMCPServerManager:
|
|||
host_progress_callback=None,
|
||||
allow_input_required=False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch=None,
|
||||
):
|
||||
# Return a mock CallToolResult
|
||||
result = MagicMock(spec=CallToolResult)
|
||||
|
|
@ -7682,6 +7683,30 @@ class TestMCPServerManager:
|
|||
# Verify the MCP client call was awaited exactly once
|
||||
assert mock_client.call_tool.await_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_hands_on_dispatch_to_the_mcp_client(self):
|
||||
"""The MCP client fires on_dispatch when it sends the call, so call_tool must hand it down."""
|
||||
from mcp.types import CallToolResult
|
||||
|
||||
auth = self._unrestricted_auth()
|
||||
manager, proxy_logging_obj = self._manager_ready_for_call_tool(
|
||||
[MCPTool(name="test_tool", inputSchema={"type": "object"})],
|
||||
caller=ListedToolsCaller(user_api_key_auth=auth),
|
||||
)
|
||||
manager.create_mcp_client = AsyncMock(return_value=_RetryFakeClient(result=CallToolResult(content=[])))
|
||||
dispatched = MagicMock()
|
||||
|
||||
await manager.call_tool(
|
||||
server_name="test-server",
|
||||
name="test_tool",
|
||||
arguments={},
|
||||
user_api_key_auth=auth,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
on_dispatch=dispatched,
|
||||
)
|
||||
|
||||
dispatched.assert_called_once_with()
|
||||
|
||||
@staticmethod
|
||||
def _manager_ready_for_call_tool(
|
||||
listed_tools: list[MCPTool], caller: ListedToolsCaller | None = None
|
||||
|
|
@ -12179,9 +12204,12 @@ class _RetryFakeClient:
|
|||
raise_on_error=False,
|
||||
allow_input_required=False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch=None,
|
||||
):
|
||||
self.attempts += 1
|
||||
self.persistent_sessions.append(persistent_session)
|
||||
if on_dispatch is not None:
|
||||
on_dispatch()
|
||||
if self._raises is not None:
|
||||
if raise_on_error:
|
||||
raise self._raises
|
||||
|
|
@ -12319,6 +12347,56 @@ class TestOBOCallToolRetry:
|
|||
manager._cred_provider.invalidate_credentials.assert_awaited_once()
|
||||
assert first.attempts == 1 and retry.attempts == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ["plain", "relay", "obo_retry"])
|
||||
async def test_call_regular_reports_every_sent_attempt_through_on_dispatch(self, path):
|
||||
"""on_dispatch tells the caller a tool may have run, so every attempt that is sent must reach
|
||||
it: once on the plain and relay paths, twice when the OBO path retries after an upstream 401."""
|
||||
manager = self._manager()
|
||||
success = CallToolResult(content=[], isError=False)
|
||||
clients = [_RetryFakeClient(raises=_UpstreamAuthError(401)), _RetryFakeClient(result=success)]
|
||||
manager.create_mcp_client = AsyncMock(side_effect=clients if path == "obo_retry" else clients[1:])
|
||||
servers = {
|
||||
"plain": MCPServer(
|
||||
server_id="plain-srv", name="plain", url="https://upstream.example/mcp", transport=MCPTransport.sse
|
||||
),
|
||||
"relay": MCPServer(
|
||||
server_id="relay-srv",
|
||||
name="relay",
|
||||
url="https://upstream.example/mcp",
|
||||
transport=MCPTransport.sse,
|
||||
auth_type=MCPAuth.true_passthrough,
|
||||
),
|
||||
"obo_retry": MCPServer(
|
||||
server_id="id-jag-srv",
|
||||
name="id-jag",
|
||||
url="https://upstream.example/mcp",
|
||||
transport=MCPTransport.sse,
|
||||
auth_type=MCPAuth.oauth2_id_jag,
|
||||
client_id="gateway-client",
|
||||
client_secret="gateway-secret",
|
||||
token_exchange_endpoint="https://org-idp.example/oauth2/token",
|
||||
id_jag_resource_token_endpoint="https://resource-as.example/oauth2/token",
|
||||
),
|
||||
}
|
||||
dispatched = MagicMock()
|
||||
|
||||
result = await manager._call_regular_mcp_tool(
|
||||
mcp_server=servers[path],
|
||||
original_tool_name="tool",
|
||||
arguments={},
|
||||
tasks=[],
|
||||
mcp_auth_header=None,
|
||||
mcp_server_auth_headers=None,
|
||||
oauth2_headers={"Authorization": "Bearer caller-id-token"} if path == "obo_retry" else None,
|
||||
raw_headers={"authorization": "Bearer caller-token"} if path == "relay" else None,
|
||||
proxy_logging_obj=None,
|
||||
on_dispatch=dispatched,
|
||||
)
|
||||
|
||||
assert result is success
|
||||
assert dispatched.call_count == (2 if path == "obo_retry" else 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_auth_error_does_not_retry(self):
|
||||
manager = self._manager()
|
||||
|
|
@ -12464,6 +12542,7 @@ class TestOBOConcurrencyLimit:
|
|||
raise_on_error=False,
|
||||
allow_input_required=False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch=None,
|
||||
):
|
||||
inflight["current"] += 1
|
||||
inflight["peak"] = max(inflight["peak"], inflight["current"])
|
||||
|
|
@ -12548,6 +12627,7 @@ class TestOBOConcurrencyLimit:
|
|||
raise_on_error: bool = False,
|
||||
allow_input_required: bool = False,
|
||||
persistent_session: PersistentMCPSession | None = None,
|
||||
on_dispatch: Callable[[], None] | None = None,
|
||||
) -> CallToolResult:
|
||||
sessions_seen.append(persistent_session)
|
||||
return CallToolResult(content=[], isError=False)
|
||||
|
|
|
|||
|
|
@ -12,14 +12,19 @@ Regression tests for https://github.com/BerriAI/litellm/issues/21343
|
|||
|
||||
import asyncio
|
||||
import datetime
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Final
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Final, NamedTuple
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
||||
_union_duration_ms,
|
||||
|
|
@ -27,8 +32,14 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
|||
)
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.litellm_core_utils.rules import Rules
|
||||
from litellm.litellm_core_utils.mcp_tool_replay import mark_mcp_tools_executed
|
||||
from litellm.types.router import RetryPolicy
|
||||
from litellm.utils import function_setup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
|
||||
def _make_rate_limit_error(message="Rate limited"):
|
||||
"""Create a RateLimitError for testing."""
|
||||
|
|
@ -66,7 +77,7 @@ def _make_not_found_error(message="Model not found"):
|
|||
)
|
||||
|
||||
|
||||
def _create_router(num_retries=2):
|
||||
def _create_router(num_retries=2, retry_policy=None):
|
||||
"""Create a Router with two deployments for testing."""
|
||||
return Router(
|
||||
model_list=[
|
||||
|
|
@ -86,6 +97,7 @@ def _create_router(num_retries=2):
|
|||
},
|
||||
],
|
||||
num_retries=num_retries,
|
||||
retry_policy=retry_policy,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -357,3 +369,512 @@ async def test_retry_attempts_accumulate_timing_in_shared_request_metadata():
|
|||
assert timing_metrics["litellm_overhead_time_ms"] == pytest.approx(
|
||||
round(total_response_time_ms - union_duration_ms, 4)
|
||||
)
|
||||
|
||||
|
||||
def _make_follow_up_error(tools_executed: bool) -> litellm.InternalServerError:
|
||||
error: Final = litellm.InternalServerError(message="follow-up failed", llm_provider="openai", model="gpt-4")
|
||||
if tools_executed:
|
||||
mark_mcp_tools_executed(error)
|
||||
return error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("retry_policy", [None, RetryPolicy(InternalServerErrorRetries=2)])
|
||||
@pytest.mark.parametrize("tools_executed", [True, False])
|
||||
async def test_500_after_mcp_tool_execution_is_not_retried(retry_policy, tools_executed):
|
||||
"""
|
||||
Regression test for #43153: a retry re-runs the whole MCP gateway loop, so a
|
||||
failure raised after the loop executed tool calls must surface instead of
|
||||
executing those tools again. A retry policy that retries 500s must not override
|
||||
that, and a 500 raised before any tool ran must still be retried.
|
||||
"""
|
||||
router: Final = _create_router(num_retries=2, retry_policy=retry_policy)
|
||||
error: Final = _make_follow_up_error(tools_executed)
|
||||
attempt: Final = AsyncMock(side_effect=error)
|
||||
|
||||
with pytest.raises(litellm.InternalServerError) as exc_info:
|
||||
await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 2})
|
||||
|
||||
assert exc_info.value is error
|
||||
assert attempt.await_count == (1 if tools_executed else 3)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_500_after_mcp_tool_execution_stops_an_ongoing_retry_loop():
|
||||
"""
|
||||
The first attempt fails before any tool ran and is retried; the retry executes
|
||||
the tools and then fails, so the loop must stop there instead of running them
|
||||
a second time.
|
||||
"""
|
||||
router: Final = _create_router(num_retries=3)
|
||||
after_tools: Final = _make_follow_up_error(tools_executed=True)
|
||||
attempt: Final = AsyncMock(side_effect=[_make_follow_up_error(tools_executed=False), after_tools])
|
||||
|
||||
with pytest.raises(litellm.InternalServerError) as exc_info:
|
||||
await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 3})
|
||||
|
||||
assert exc_info.value is after_tools
|
||||
assert attempt.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tools_executed", [True, False])
|
||||
async def test_500_after_mcp_tool_execution_does_not_fall_back(tools_executed):
|
||||
"""
|
||||
A fallback re-runs the whole MCP gateway loop on the fallback group, so it must
|
||||
not start once the primary attempt executed tool calls.
|
||||
"""
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "test-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-1"}},
|
||||
{"model_name": "backup-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-2"}},
|
||||
],
|
||||
num_retries=0,
|
||||
fallbacks=[{"test-model": ["backup-model"]}],
|
||||
)
|
||||
error: Final = _make_follow_up_error(tools_executed)
|
||||
fallback_response: Final = litellm.ModelResponse()
|
||||
|
||||
async def primary_fails(**kwargs):
|
||||
if kwargs["model"] == "test-model":
|
||||
raise error
|
||||
return fallback_response
|
||||
|
||||
attempt: Final = AsyncMock(side_effect=primary_fails)
|
||||
call: Final = router.async_function_with_fallbacks(
|
||||
**{**_base_kwargs(), "original_function": attempt, "num_retries": 0}
|
||||
)
|
||||
|
||||
if tools_executed:
|
||||
with pytest.raises(litellm.InternalServerError) as exc_info:
|
||||
await call
|
||||
assert exc_info.value is error
|
||||
else:
|
||||
assert await call is fallback_response
|
||||
|
||||
attempted_models: Final = [awaited.kwargs["model"] for awaited in attempt.await_args_list]
|
||||
assert attempted_models == (["test-model"] if tools_executed else ["test-model", "backup-model"])
|
||||
|
||||
|
||||
_UPSTREAM: Final = "https://upstream.example/v1"
|
||||
_TOOL_NAME: Final = "ledger-commit_write"
|
||||
|
||||
|
||||
def _ledger_tools() -> list[dict[str, str]]:
|
||||
return [{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}]
|
||||
|
||||
|
||||
def _ledger_caller() -> "UserAPIKeyAuth":
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
|
||||
|
||||
return UserAPIKeyAuth(
|
||||
object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"])
|
||||
)
|
||||
|
||||
|
||||
class _LedgerGateway(NamedTuple):
|
||||
tool: AsyncMock
|
||||
list_tools: AsyncMock
|
||||
manager: "MCPServerManager"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway:
|
||||
"""Serve one auto-approved MCP tool through the real MCP manager and tool registry."""
|
||||
from mcp.types import Tool
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.mcp_server import mcp_server_manager, server, tool_registry
|
||||
from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
manager: Final = mcp_server_manager.MCPServerManager()
|
||||
manager.registry = {
|
||||
"ledger": MCPServer(
|
||||
server_id="ledger",
|
||||
name="ledger",
|
||||
server_name="ledger",
|
||||
transport="http",
|
||||
url="https://ledger.example/mcp",
|
||||
spec_path="ledger.json",
|
||||
auth_type="none",
|
||||
)
|
||||
}
|
||||
manager.tool_name_to_mcp_server_name_mapping = {_TOOL_NAME: "ledger"}
|
||||
tool: Final = AsyncMock(return_value={"written": True})
|
||||
registry: Final = tool_registry.MCPToolRegistry()
|
||||
registry.register_tool(_TOOL_NAME, "Write one ledger entry", {"type": "object"}, tool)
|
||||
list_tools: Final = AsyncMock(
|
||||
return_value=AggregateToolListing(tools=[Tool(name=_TOOL_NAME, inputSchema={"type": "object"})], outcomes={})
|
||||
)
|
||||
monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry)
|
||||
monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager)
|
||||
monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
|
||||
monkeypatch.setattr(server, "_get_tools_from_mcp_servers", list_tools)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
return _LedgerGateway(tool=tool, list_tools=list_tools, manager=manager)
|
||||
|
||||
|
||||
def _is_follow_up(request: httpx.Request) -> bool:
|
||||
body: Final = json.loads(request.content)
|
||||
messages: Final = body.get("messages") or []
|
||||
items: Final = body["input"] if isinstance(body.get("input"), list) else []
|
||||
return any(message.get("role") == "tool" for message in messages) or any(
|
||||
item.get("type") == "function_call_output" for item in items
|
||||
)
|
||||
|
||||
|
||||
def _chat_completion(message: dict[str, object], finish_reason: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-4.1-mini",
|
||||
"choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _responses_api_response(output_item: dict[str, object]) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "gpt-4.1-mini",
|
||||
"output": [output_item],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tool_call(responses_api: bool) -> httpx.Response:
|
||||
arguments: Final = json.dumps({"entry": "alpha"})
|
||||
if responses_api:
|
||||
return _responses_api_response(
|
||||
{"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": _TOOL_NAME, "arguments": arguments}
|
||||
)
|
||||
return _chat_completion(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": _TOOL_NAME, "arguments": arguments}}
|
||||
],
|
||||
},
|
||||
"tool_calls",
|
||||
)
|
||||
|
||||
|
||||
def _final_answer(responses_api: bool) -> httpx.Response:
|
||||
if responses_api:
|
||||
return _responses_api_response(
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Recorded alpha", "annotations": []}],
|
||||
}
|
||||
)
|
||||
return _chat_completion({"role": "assistant", "content": "Recorded alpha"}, "stop")
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
"""Answers both OpenAI routes: a tool call, then a final answer once the tool result arrives."""
|
||||
|
||||
def __init__(self, failing_initial_calls: int = 0, failing_follow_ups: int = 0) -> None:
|
||||
self.calls: list[str] = []
|
||||
self._failures: Final = {"initial": failing_initial_calls, "follow_up": failing_follow_ups}
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
kind: Final = "follow_up" if _is_follow_up(request) else "initial"
|
||||
self.calls.append(kind)
|
||||
if self.calls.count(kind) <= self._failures[kind]:
|
||||
return httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}})
|
||||
responses_api: Final = request.url.path.endswith("/responses")
|
||||
return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api)
|
||||
|
||||
|
||||
class _StreamBrokenBeforeFirstChunk(httpx.AsyncByteStream):
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
raise httpx.ReadError("connection dropped before the first chunk")
|
||||
yield b""
|
||||
|
||||
|
||||
def _chat_tool_call_stream() -> httpx.Response:
|
||||
tool_call: Final = {
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": _TOOL_NAME, "arguments": json.dumps({"entry": "alpha"})},
|
||||
}
|
||||
chunks: Final = (
|
||||
{"role": "assistant", "content": None, "tool_calls": [tool_call]},
|
||||
{},
|
||||
)
|
||||
body: Final = "".join(
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "gpt-4.1-mini",
|
||||
"choices": [{"index": 0, "delta": delta, "finish_reason": None if delta else "tool_calls"}],
|
||||
}
|
||||
)
|
||||
+ "\n\n"
|
||||
for delta in chunks
|
||||
)
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=f"{body}data: [DONE]\n\n")
|
||||
|
||||
|
||||
def _serve(model: _FakeModel) -> None:
|
||||
respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model)
|
||||
respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model)
|
||||
|
||||
|
||||
def _mcp_router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router:
|
||||
deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM}
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "repro-model", "litellm_params": deployment},
|
||||
{"model_name": "repro-model", "litellm_params": deployment},
|
||||
{"model_name": "backup-model", "litellm_params": deployment},
|
||||
],
|
||||
num_retries=num_retries,
|
||||
fallbacks=fallbacks,
|
||||
)
|
||||
|
||||
|
||||
def _chat(router: litellm.Router) -> Awaitable[object]:
|
||||
return router.acompletion(
|
||||
model="repro-model",
|
||||
messages=[{"role": "user", "content": "Record entry alpha"}],
|
||||
tools=_ledger_tools(),
|
||||
metadata={"user_api_key_auth": _ledger_caller()},
|
||||
)
|
||||
|
||||
|
||||
def _responses(router: litellm.Router) -> Awaitable[object]:
|
||||
return router.aresponses(
|
||||
model="repro-model",
|
||||
input="Record entry alpha",
|
||||
tools=_ledger_tools(),
|
||||
litellm_metadata={"user_api_key_auth": _ledger_caller()},
|
||||
)
|
||||
|
||||
|
||||
def _messages(router: litellm.Router) -> Awaitable[object]:
|
||||
return router.aanthropic_messages(
|
||||
model="repro-model",
|
||||
max_tokens=64,
|
||||
messages=[{"role": "user", "content": "Record entry alpha"}],
|
||||
tools=_ledger_tools(),
|
||||
litellm_metadata={"user_api_key_auth": _ledger_caller()},
|
||||
)
|
||||
|
||||
|
||||
_ENDPOINTS: Final = pytest.mark.parametrize(
|
||||
"send", [_chat, _responses, _messages], ids=["chat_completions", "responses", "messages"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_ENDPOINTS
|
||||
@pytest.mark.parametrize(
|
||||
"router_kwargs",
|
||||
[{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}],
|
||||
ids=["retries", "fallbacks"],
|
||||
)
|
||||
@respx.mock
|
||||
async def test_router_does_not_replay_an_executed_mcp_tool(
|
||||
ledger_gateway: _LedgerGateway,
|
||||
send: Callable[[litellm.Router], Awaitable[object]],
|
||||
router_kwargs: dict[str, object],
|
||||
):
|
||||
"""
|
||||
Regression test for #43153: an auto-approved MCP tool must run once per request.
|
||||
|
||||
Given: The model asks for the tool, the tool runs, and every follow-up model call returns a 500
|
||||
When: The router has retries or a fallback group configured
|
||||
Then: The client gets the 500 after one tool execution and one follow-up call
|
||||
|
||||
A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed
|
||||
the tool again (three writes with num_retries=2, two with one fallback).
|
||||
"""
|
||||
model: Final = _FakeModel(failing_follow_ups=3)
|
||||
_serve(model)
|
||||
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
await send(_mcp_router(**router_kwargs))
|
||||
|
||||
assert ledger_gateway.tool.await_count == 1
|
||||
assert model.calls == ["initial", "follow_up"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_ENDPOINTS
|
||||
@respx.mock
|
||||
async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool(
|
||||
ledger_gateway: _LedgerGateway,
|
||||
send: Callable[[litellm.Router], Awaitable[object]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
Given: A pre-call guardrail blocks the tool call, so no tool runs
|
||||
When: The follow-up model call fails once with a 500
|
||||
Then: The router still retries, because replaying the request cannot repeat a side effect
|
||||
"""
|
||||
|
||||
class BlockEveryToolCall(CustomGuardrail):
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
raise GuardrailRaisedException(message="tool calls are blocked", blocked_content=True)
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm, "callbacks", [BlockEveryToolCall(guardrail_name="block", event_hook="pre_mcp_call", default_on=True)]
|
||||
)
|
||||
model: Final = _FakeModel(failing_follow_ups=1)
|
||||
_serve(model)
|
||||
|
||||
await send(_mcp_router(num_retries=2))
|
||||
|
||||
assert ledger_gateway.tool.await_count == 0
|
||||
assert model.calls == ["initial", "follow_up", "initial", "follow_up"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_ENDPOINTS
|
||||
@respx.mock
|
||||
async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent(
|
||||
ledger_gateway: _LedgerGateway,
|
||||
send: Callable[[litellm.Router], Awaitable[object]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
Given: The tool runs, but its call fails afterwards, here because a during-call guardrail
|
||||
rejects the result, which leaves the router in the same spot as a timeout after the write
|
||||
When: Every follow-up model call returns a 500
|
||||
Then: The router does not replay the request, because the tool may already have written
|
||||
"""
|
||||
tool_ran: Final = asyncio.Event()
|
||||
|
||||
def write(**_: object) -> dict[str, bool]:
|
||||
tool_ran.set()
|
||||
return {"written": True}
|
||||
|
||||
class RejectToolResult(CustomGuardrail):
|
||||
async def async_moderation_hook(self, data, user_api_key_dict, call_type):
|
||||
await tool_ran.wait()
|
||||
raise GuardrailRaisedException(message="tool result rejected", blocked_content=True)
|
||||
|
||||
ledger_gateway.tool.side_effect = write
|
||||
monkeypatch.setattr(
|
||||
litellm, "callbacks", [RejectToolResult(guardrail_name="reject", event_hook="during_mcp_call", default_on=True)]
|
||||
)
|
||||
model: Final = _FakeModel(failing_follow_ups=3)
|
||||
_serve(model)
|
||||
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
await send(_mcp_router(num_retries=2))
|
||||
|
||||
assert ledger_gateway.tool.await_count == 1
|
||||
assert model.calls == ["initial", "follow_up"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_router_retries_when_an_mcp_tool_call_never_reached_its_server(
|
||||
ledger_gateway: _LedgerGateway,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
Given: The MCP server needs a per-user header the caller has not set, so the tool call fails
|
||||
before anything is sent to the server
|
||||
When: The follow-up model call fails once with a 500
|
||||
Then: The router still retries, because replaying the request cannot repeat a side effect
|
||||
"""
|
||||
registry: Final = ledger_gateway.manager.registry
|
||||
monkeypatch.setitem(
|
||||
registry,
|
||||
"ledger",
|
||||
registry["ledger"].model_copy(
|
||||
update={
|
||||
"spec_path": None,
|
||||
"static_headers": {"Authorization": "Bearer ${LEDGER_TOKEN}"},
|
||||
"env_vars": [{"name": "LEDGER_TOKEN", "scope": "user"}],
|
||||
}
|
||||
),
|
||||
)
|
||||
model: Final = _FakeModel(failing_follow_ups=1)
|
||||
_serve(model)
|
||||
|
||||
await _chat(_mcp_router(num_retries=2))
|
||||
|
||||
assert model.calls == ["initial", "follow_up", "initial", "follow_up"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran(
|
||||
ledger_gateway: _LedgerGateway,
|
||||
):
|
||||
"""
|
||||
Given: A streaming Chat Completions request whose tool ran
|
||||
When: The follow-up stream breaks before its first chunk and a fallback group is configured
|
||||
Then: The client gets the error instead of a fallback that runs the tool again
|
||||
"""
|
||||
calls: Final[list[str]] = []
|
||||
|
||||
def stream_model(request: httpx.Request) -> httpx.Response:
|
||||
kind: Final = "follow_up" if _is_follow_up(request) else "initial"
|
||||
calls.append(kind)
|
||||
if kind == "follow_up":
|
||||
return httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"}, stream=_StreamBrokenBeforeFirstChunk()
|
||||
)
|
||||
return _chat_tool_call_stream()
|
||||
|
||||
respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=stream_model)
|
||||
stream: Final = await _mcp_router(num_retries=0, fallbacks=[{"repro-model": ["backup-model"]}]).acompletion(
|
||||
model="repro-model",
|
||||
messages=[{"role": "user", "content": "Record entry alpha"}],
|
||||
tools=_ledger_tools(),
|
||||
metadata={"user_api_key_auth": _ledger_caller()},
|
||||
stream=True,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
assert ledger_gateway.tool.await_count == 1
|
||||
assert calls == ["initial", "follow_up"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway):
|
||||
"""
|
||||
Given: The initial model call fails once with a 500, then asks for the tool
|
||||
When: The router retries
|
||||
Then: The retry succeeds and the tool runs once
|
||||
"""
|
||||
model: Final = _FakeModel(failing_initial_calls=1)
|
||||
_serve(model)
|
||||
|
||||
await _chat(_mcp_router(num_retries=2))
|
||||
|
||||
assert ledger_gateway.tool.await_count == 1
|
||||
assert model.calls == ["initial", "initial", "follow_up"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue