From aa48016d913e1240978b53ccc007f9ecaa3c482a Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 14:43:14 +1000 Subject: [PATCH 1/6] fix(responses): surface MCP gateway initial-call failures instead of emitting a broken stream When the initial LLM call inside MCPEnhancedStreamingIterator fails (e.g. an invalid previous_response_id -> provider 400 'No tool output found for function call ...'), the proxy returned HTTP 200 and the stream emitted the pre-generated mcp_list_tools discovery events with no response.created before them. That violates the Responses API streaming contract and crashes SDK stream accumulators (openai-node: "expected 'response.created' event, got response.mcp_list_tools.in_progress"). - aresponses_api_with_mcp now makes the initial call eagerly, before any SSE bytes are written, and re-raises the stashed failure so the client gets a real 4xx/5xx with the provider error body. - If a creation failure still surfaces during iteration, the stream emits a single terminal 'error' event instead of discovery events. Co-Authored-By: Claude Fable 5 --- litellm/proxy/dev_config.yaml | 5 + litellm/responses/main.py | 12 +- .../responses/mcp/mcp_streaming_iterator.py | 52 +++++-- .../mcp/test_mcp_streaming_iterator.py | 145 ++++++++++++++++++ 4 files changed, 204 insertions(+), 10 deletions(-) create mode 100644 tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml index 65a4b8e7cbf..8ad7da6d21a 100644 --- a/litellm/proxy/dev_config.yaml +++ b/litellm/proxy/dev_config.yaml @@ -211,3 +211,8 @@ litellm_settings: sandbox_tool_name: e2b_sandbox callbacks: - code_interpreter_interception + +mcp_servers: + deepwiki: + url: "https://mcp.deepwiki.com/mcp" + transport: "http" diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8e3be2bc12d..0f9e3d86ece 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -261,7 +261,7 @@ async def aresponses_api_with_mcp( pre_processed_mcp_tools=original_mcp_tools, ) - return LiteLLM_Proxy_MCP_Handler._create_mcp_streaming_response( + mcp_streaming_response = LiteLLM_Proxy_MCP_Handler._create_mcp_streaming_response( input=input, model=model, all_tools=all_tools, @@ -272,6 +272,16 @@ async def aresponses_api_with_mcp( tool_server_map=tool_server_map, **kwargs, ) + # Make the initial LLM call eagerly, before any SSE bytes are written, + # so a pre-stream failure (e.g. an invalid previous_response_id -> + # provider 400 "No tool output found for function call ...") surfaces + # as a normal HTTP error instead of an HTTP 200 whose stream emits + # mcp_list_tools events with no response.created (which crashes SDK + # stream accumulators). + await mcp_streaming_response._create_initial_response_iterator() + if mcp_streaming_response._initial_creation_error is not None: + raise mcp_streaming_response._initial_creation_error + return mcp_streaming_response # Determine if we should auto-execute tools should_auto_execute = bool(mcp_tools_with_litellm_proxy) and LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools( diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 3c24a703b68..02a978c9445 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -5,6 +5,8 @@ from litellm._uuid import uuid from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, + ErrorEvent, + ErrorEventError, MCPCallArgumentsDeltaEvent, MCPCallArgumentsDoneEvent, MCPCallCompletedEvent, @@ -304,6 +306,14 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Cache the response ID to ensure consistency across all events self._cached_response_id: Optional[str] = None + # Initial-LLM-call failures are stashed here so they can be surfaced + # to the client as an `error` stream event (lazy path) or re-raised + # before any SSE bytes are written (eager path in + # aresponses_api_with_mcp). + self._initial_creation_error: Optional[Exception] = None + self._stream_error: Optional[Exception] = None + self._error_event_emitted = False + def _extract_mcp_headers_from_params(self) -> None: """Extract MCP headers from original request params to pass to tool calls""" from typing import Dict, Optional @@ -368,6 +378,23 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): return LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(self.mcp_tools_with_litellm_proxy) + def _make_stream_error_event(self) -> ResponsesAPIStreamingResponse: + """Build an OpenAI-style `error` stream event from the stashed internal + failure, so clients receive a real terminal error instead of a stream + that silently ends mid-flow.""" + err = self._stream_error + status_code = getattr(err, "status_code", None) + return ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=1, + error=ErrorEventError( + type="mcp_gateway_error", + code=str(status_code) if status_code is not None else "internal_error", + message=str(err) if err is not None else "MCP gateway stream failed", + param=None, + ), + ) + def __aiter__(self): return self @@ -451,13 +478,17 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): await self._create_initial_response_iterator() if self.base_iterator is None: - # LLM call failed — still emit MCP discovery events before finishing - if self.mcp_discovery_events: - self.phase = "mcp_discovery" - else: - self.phase = "finished" - raise StopAsyncIteration - return None + # The initial LLM call failed. Do NOT emit MCP discovery events: a + # stream that starts with mcp_list_tools events and no + # response.created violates the Responses API streaming contract + # and crashes SDK stream accumulators (openai-node: "expected + # 'response.created' event, got response.mcp_list_tools.in_progress"). + # Surface the failure as an `error` event instead. + self.phase = "finished" + if self._stream_error is not None: + self._error_event_emitted = True + return self._make_stream_error_event() + raise StopAsyncIteration if self.base_iterator: if hasattr(self.base_iterator, "__anext__"): @@ -580,8 +611,11 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): traceback.print_exc() self.base_iterator = None - # Don't set phase to "finished" here — let __anext__ emit any - # pre-generated MCP discovery events before ending the iteration. + # Stash the failure so aresponses_api_with_mcp can re-raise it + # before any SSE bytes are written (eager creation), or so + # __anext__ can emit an `error` event instead of ending silently. + self._initial_creation_error = e + self._stream_error = e async def _generate_tool_execution_events(self) -> None: """Generate tool execution events and execute tools""" diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py new file mode 100644 index 00000000000..a9de6ebd4dd --- /dev/null +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -0,0 +1,145 @@ +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +import litellm +from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator +from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamEvents + +# `litellm.__init__` re-exports a function named `responses`, which shadows the +# `litellm.responses` subpackage as an attribute — `import litellm.responses.main` +# can resolve to the unrelated third-party `responses` package instead. Look the +# real submodule up in sys.modules directly to sidestep the shadowing. +responses_main_module = sys.modules["litellm.responses.main"] + + +class _FakeAsyncStream: + """Minimal async iterator yielding pre-built chunks, one per __anext__ call.""" + + def __init__(self, chunks): + self._chunks = list(chunks) + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._chunks: + raise StopAsyncIteration + return self._chunks.pop(0) + + +def _completed_chunk(output): + response = ResponsesAPIResponse(id="resp-1", created_at=0, output=output) + return SimpleNamespace(type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=response) + + +def _text_message(text: str): + return {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]} + + +def _text_only_stream(text: str) -> _FakeAsyncStream: + return _FakeAsyncStream([_completed_chunk([_text_message(text)])]) + + +def _make_lazy_iterator(mcp_events=None) -> MCPEnhancedStreamingIterator: + """Iterator with no base_iterator: the initial LLM call happens lazily on iteration.""" + return MCPEnhancedStreamingIterator( + base_iterator=None, + mcp_events=list(mcp_events or []), + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-4", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + }, + ) + + +@pytest.mark.asyncio +async def test_initial_call_failure_emits_error_event_not_discovery_events(monkeypatch): + """ + Regression test: when the initial LLM call fails (e.g. an invalid + previous_response_id -> provider 400 "No tool output found for function + call ..."), the stream used to emit the pre-generated mcp_list_tools + discovery events with no response.created before them — which violates + the Responses API streaming contract and crashes SDK stream accumulators + (openai-node: "expected 'response.created' event, got + response.mcp_list_tools.in_progress"). The stream must instead surface a + single terminal `error` event and end. + """ + aresponses_mock = AsyncMock( + side_effect=litellm.BadRequestError( + message="No tool output found for function call call_x.", + model="gpt-4", + llm_provider="openai", + ) + ) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + discovery_event = SimpleNamespace(type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS) + iterator = _make_lazy_iterator(mcp_events=[discovery_event]) + + chunks = [chunk async for chunk in iterator] + + assert len(chunks) == 1 + error_event = chunks[0] + assert error_event.type == ResponsesAPIStreamEvents.ERROR + assert error_event.error.code == "400" + assert "No tool output found" in error_event.error.message + # No discovery events leaked before/after the error. + assert discovery_event not in chunks + + +@pytest.mark.asyncio +async def test_eager_creation_reraises_pre_stream_failure_as_http_error(monkeypatch): + """ + aresponses_api_with_mcp creates the initial response eagerly and re-raises + the stashed creation failure, so the proxy returns a real 4xx/5xx before + any SSE bytes are written instead of an HTTP 200 with a broken stream. + """ + from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([], {})), + ) + boom = litellm.BadRequestError( + message="Previous response with id 'resp_bogus' not found.", + model="gpt-4", + llm_provider="openai", + ) + monkeypatch.setattr(responses_main_module, "aresponses", AsyncMock(side_effect=boom)) + + with pytest.raises(litellm.BadRequestError) as excinfo: + await responses_main_module.aresponses_api_with_mcp( + input="hi", + model="gpt-4", + stream=True, + previous_response_id="resp_bogus", + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/deepwiki", "require_approval": "never"}], + ) + + assert "resp_bogus" in str(excinfo.value) + + +@pytest.mark.asyncio +async def test_initial_call_success_does_not_emit_error_event(monkeypatch): + """Happy path is unchanged: no error event, stream flows as before.""" + monkeypatch.setattr( + responses_main_module, + "aresponses", + AsyncMock(return_value=_text_only_stream("all good")), + ) + + iterator = _make_lazy_iterator() + chunks = [chunk async for chunk in iterator] + + assert all(getattr(c, "type", None) != ResponsesAPIStreamEvents.ERROR for c in chunks) + completed = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + assert len(completed) == 1 + assert iterator._initial_creation_error is None From 70656be89e77330e3579a050b20eecebd30d3f59 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 14:44:36 +1000 Subject: [PATCH 2/6] fix(responses): emit terminal error event on MCP tool-execution / follow-up failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When tool execution failed as a batch, the stream proceeded to a follow-up call carrying function_call items with no outputs — rejected by the provider with 'No tool output found for function call ...' — and when the follow-up call itself failed, the stream simply ended with no terminal event. In both cases the client received HTTP 200 and a stream that looks like a truncated success: tool events, then silence. - Stash tool-execution and follow-up failures on the iterator. - Skip the doomed follow-up call entirely after a tool-execution failure. - Emit a single terminal OpenAI-style 'error' stream event carrying the mapped failure instead of ending silently. Builds on the initial-call failure handling from the previous commit (shares the _stream_error stash and _make_stream_error_event helper). Co-Authored-By: Claude Fable 5 --- .../responses/mcp/mcp_streaming_iterator.py | 27 +++- .../mcp/test_mcp_streaming_iterator.py | 127 ++++++++++++++++++ 2 files changed, 150 insertions(+), 4 deletions(-) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 02a978c9445..dae7bcbd085 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -306,10 +306,10 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Cache the response ID to ensure consistency across all events self._cached_response_id: Optional[str] = None - # Initial-LLM-call failures are stashed here so they can be surfaced - # to the client as an `error` stream event (lazy path) or re-raised - # before any SSE bytes are written (eager path in - # aresponses_api_with_mcp). + # Internal failures (initial LLM call, tool execution, follow-up call) + # are stashed here so they can be surfaced to the client as an `error` + # stream event, or re-raised before any SSE bytes are written (eager + # path in aresponses_api_with_mcp for the initial call). self._initial_creation_error: Optional[Exception] = None self._stream_error: Optional[Exception] = None self._error_event_emitted = False @@ -456,6 +456,12 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): raise else: self.phase = "finished" + # Tool execution or the follow-up call failed: emit a terminal + # `error` event so the client can distinguish a failed stream + # from a completed one. + if self._stream_error is not None and not self._error_event_emitted: + self._error_event_emitted = True + return self._make_stream_error_event() raise StopAsyncIteration # Phase 6: Finished @@ -727,11 +733,21 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): traceback.print_exc() self.tool_results = [] + # Remember the failure. Without this, the follow-up call is made + # with function_call items but no function_call_output items and + # the provider rejects it with "No tool output found for function + # call ...". + self._stream_error = e async def _create_follow_up_iterator(self) -> None: """Create the follow-up response iterator with tool results""" if not self.collected_response or not hasattr(self, "tool_results"): return + # Tool execution already failed; skip the doomed follow-up call (it + # would be rejected with "No tool output found for function call ...") + # and let __anext__ emit the terminal error event. + if self._stream_error is not None: + return from litellm.responses.main import aresponses from litellm.responses.mcp.litellm_proxy_mcp_handler import ( @@ -772,6 +788,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): traceback.print_exc() self.follow_up_iterator = None + # Surface via a terminal `error` event in __anext__ instead of + # silently ending the stream with no terminal event. + self._stream_error = e def __iter__(self): return self diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index a9de6ebd4dd..d2e593b4896 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -143,3 +143,130 @@ async def test_initial_call_success_does_not_emit_error_event(monkeypatch): completed = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] assert len(completed) == 1 assert iterator._initial_creation_error is None + + +import types +from unittest.mock import MagicMock + +from mcp.types import CallToolResult, TextContent + + +def _output_item_added_chunk(): + return SimpleNamespace(type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) + + +def _function_call(call_id: str, name: str, arguments: str = "{}"): + return {"type": "function_call", "call_id": call_id, "name": name, "arguments": arguments} + + +def _mock_mcp_environment(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: + """Patch the MCP tool-call plumbing so _execute_tool_calls can run in tests.""" + call_tool = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="ok")], isError=False)) + fake_manager = types.SimpleNamespace( + call_tool=call_tool, + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + types.SimpleNamespace(proxy_logging_obj=MagicMock()), + ) + return call_tool + + +def _make_tool_call_iterator() -> MCPEnhancedStreamingIterator: + return MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-4", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + }, + ) + + +@pytest.mark.asyncio +async def test_tool_execution_failure_emits_error_event_and_skips_follow_up(monkeypatch): + """ + When tool execution blows up as a batch (not a per-tool error string), + the stream used to proceed to a follow-up call carrying function_call + items with no outputs — rejected by the provider with "No tool output + found for function call ..." — and then end silently. It must instead + skip the doomed follow-up and emit a terminal `error` event. + """ + from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + AsyncMock(side_effect=RuntimeError("mcp server exploded")), + ) + aresponses_mock = AsyncMock() + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = _make_tool_call_iterator() + chunks = [chunk async for chunk in iterator] + + error_events = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.ERROR] + assert len(error_events) == 1 + assert "mcp server exploded" in error_events[0].error.message + # The doomed follow-up call was never made. + aresponses_mock.assert_not_called() + + +@pytest.mark.asyncio +async def test_follow_up_failure_emits_error_event(monkeypatch): + """ + When the follow-up LLM call after successful tool execution fails, the + stream used to end with no terminal event (the client saw tool events + and then... nothing). It must emit a terminal `error` event carrying the + mapped provider failure. + """ + _mock_mcp_environment(monkeypatch) + + boom = litellm.BadRequestError( + message="No tool output found for function call call_1.", + model="gpt-4", + llm_provider="openai", + ) + monkeypatch.setattr(responses_main_module, "aresponses", AsyncMock(side_effect=boom)) + + iterator = _make_tool_call_iterator() + chunks = [chunk async for chunk in iterator] + + error_events = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.ERROR] + assert len(error_events) == 1 + assert error_events[0].error.code == "400" + assert "No tool output found" in error_events[0].error.message + # Tool-execution events were still streamed before the error surfaced. + assert any(getattr(c, "type", None) == ResponsesAPIStreamEvents.MCP_CALL_COMPLETED for c in chunks) + + +@pytest.mark.asyncio +async def test_tool_call_happy_path_emits_no_error_event(monkeypatch): + """Regression guard: the tool-call success path must stay error-free.""" + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(return_value=_text_only_stream("final answer")) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = _make_tool_call_iterator() + chunks = [chunk async for chunk in iterator] + + assert all(getattr(c, "type", None) != ResponsesAPIStreamEvents.ERROR for c in chunks) + completed = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + assert completed[-1].response.output[0]["content"][0]["text"] == "final answer" From b6d9d85f3ef43c408446143957b11d507c31a0e2 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 14:48:15 +1000 Subject: [PATCH 3/6] fix(responses): reject MCP gateway requests that resolve zero tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A request that explicitly asks for MCP tools via server_url litellm_proxy/... but resolves none of them (the API key/team has no access to the MCP server via allow_all_keys=false and no object-permission grant, the server name does not exist, or allowed_tools matches nothing) was silently sent to the model with no tools. The model then hallucinates, and the only trace is a list_mcp_tools spend log with status success and an empty response — the request looks healthy end to end while being completely broken. Raise a 400 BadRequestError naming the requested server URLs and the likely causes instead. Guard scope: - Mixed requests are exempt: with other (function) tools present, the request proceeds using those tools, matching the previous fallback behaviour. - Opt-out via litellm.reject_empty_mcp_resolved_tools = False (default True, per maintainer guidance). The auth-header pass-through test in tests/mcp_tests now resolves a dummy tool, since its purpose is header propagation, not zero-tool behaviour. Co-Authored-By: Claude Fable 5 --- litellm/__init__.py | 6 + litellm/responses/main.py | 29 ++++ .../mcp_tests/test_aresponses_api_with_mcp.py | 7 +- .../mcp/test_mcp_empty_resolved_tools.py | 144 ++++++++++++++++++ .../mcp/test_mcp_streaming_iterator.py | 5 +- 5 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 9327e121b1d..59bc8c37fd2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -316,6 +316,12 @@ disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False disable_vertex_batch_output_transformation: bool = False +# Raise a 400 when a Responses API request asks for MCP gateway tools +# (server_url litellm_proxy/...) but zero tools resolve (key/team lacks server +# access, unknown server name, or allowed_tools matches nothing) and the +# request carries no other tools. Without this the model is silently called +# with no tools and hallucinates. Set to False to restore the old behaviour. +reject_empty_mcp_resolved_tools: bool = True extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 0f9e3d86ece..64528c7dd0a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -216,6 +216,35 @@ async def aresponses_api_with_mcp( ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools) + if ( + litellm.reject_empty_mcp_resolved_tools + and mcp_tools_with_litellm_proxy + and not original_mcp_tools + and not other_tools + ): + # The request explicitly asked for MCP tools but none resolved, and + # there are no other tools to fall back on. This is almost always a + # misconfiguration: the API key/team has no access to the MCP server + # (allow_all_keys=false and no object-permission grant), the server + # name does not exist, or allowed_tools matches no tool on the server. + # Silently calling the model with no tools makes it hallucinate, and + # the only trace is a list_mcp_tools spend log with an empty response — + # so fail loudly instead. + requested_mcp_urls = [tool.get("server_url") for tool in mcp_tools_with_litellm_proxy if isinstance(tool, dict)] + raise litellm.BadRequestError( + message=( + "MCP gateway resolved 0 tools for the requested MCP tool(s) " + f"(server_url(s): {requested_mcp_urls}). Likely causes: the API " + "key/team does not have access to the MCP server (server has " + "allow_all_keys=false and no key/team object-permission grant), " + "the server name does not exist, or allowed_tools matches no " + "tool on the server. Set litellm.reject_empty_mcp_resolved_tools " + "= False to restore the previous silent behaviour." + ), + model=model, + llm_provider=custom_llm_provider or "openai", + ) + # Combine with other tools all_tools = openai_tools + other_tools if (openai_tools or other_tools) else None diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index 9cd45f3d6fc..cf2a9586dff 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -278,7 +278,12 @@ async def test_aresponses_api_with_mcp_passes_mcp_server_auth_headers_to_process async def mock_process(**kwargs): captured_process_kwargs.update(kwargs) - return ([], {}) + from mcp.types import Tool as MCPTool + + dummy_tool = MCPTool( + name="dummy_tool", description="dummy", inputSchema={"type": "object"} + ) + return ([dummy_tool], {"dummy_tool": "dummy_server"}) mock_response = ResponsesAPIResponse( **{ diff --git a/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py b/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py new file mode 100644 index 00000000000..1a71b14475f --- /dev/null +++ b/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py @@ -0,0 +1,144 @@ +""" +Guard tests: a request that explicitly asks for MCP tools via the litellm_proxy +gateway but resolves zero of them (and has no other tools to fall back on) must +fail loudly with a 400 instead of silently calling the model with no tools — +which makes it hallucinate, with the only trace being a "success" +list_mcp_tools spend log with an empty response. +""" + +import sys +from unittest.mock import AsyncMock + +import pytest + +import litellm +from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler +from litellm.types.llms.openai import ResponsesAPIResponse + +# See test_mcp_streaming_iterator.py: look the real submodule up in sys.modules +# to sidestep litellm.responses being shadowed by the re-exported function. +responses_main_module = sys.modules["litellm.responses.main"] + +MCP_TOOL = { + "type": "mcp", + "server_url": "litellm_proxy/mcp/nonexistent_server", + "require_approval": "never", + "allowed_tools": ["get_links"], +} + + +def _patch_resolved_tools(monkeypatch: pytest.MonkeyPatch, resolved_tools: list) -> None: + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + AsyncMock(return_value=(resolved_tools, {})), + ) + + +def _model_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse(id="resp-1", created_at=0, output=[]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_zero_resolved_mcp_tools_raises_before_model_call(monkeypatch, stream): + _patch_resolved_tools(monkeypatch, []) + aresponses_mock = AsyncMock() + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + with pytest.raises(litellm.BadRequestError) as excinfo: + await responses_main_module.aresponses_api_with_mcp( + input="how many links do i have?", + model="gpt-4", + stream=stream, + tools=[MCP_TOOL], + ) + + message = str(excinfo.value) + assert "resolved 0 tools" in message + assert "litellm_proxy/mcp/nonexistent_server" in message + assert "allow_all_keys" in message + # The model was never called without its tools. + aresponses_mock.assert_not_called() + + +@pytest.mark.asyncio +async def test_zero_resolved_mcp_tools_with_function_tools_falls_back(monkeypatch): + """Mixed requests keep working: with other (function) tools present, the + request proceeds using those tools instead of hard-failing.""" + _patch_resolved_tools(monkeypatch, []) + response = _model_response() + aresponses_mock = AsyncMock(return_value=response) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + function_tool = {"type": "function", "name": "my_fn", "parameters": {}} + result = await responses_main_module.aresponses_api_with_mcp( + input="hello", + model="gpt-4", + stream=False, + tools=[MCP_TOOL, function_tool], + ) + + assert result is response + aresponses_mock.assert_called_once() + assert aresponses_mock.call_args.kwargs["tools"] == [function_tool] + + +@pytest.mark.asyncio +async def test_zero_resolved_mcp_tools_flag_off_restores_old_behaviour(monkeypatch): + _patch_resolved_tools(monkeypatch, []) + response = _model_response() + aresponses_mock = AsyncMock(return_value=response) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + monkeypatch.setattr(litellm, "reject_empty_mcp_resolved_tools", False) + + result = await responses_main_module.aresponses_api_with_mcp( + input="how many links do i have?", + model="gpt-4", + stream=False, + tools=[MCP_TOOL], + ) + + assert result is response + aresponses_mock.assert_called_once() + + +@pytest.mark.asyncio +async def test_resolved_mcp_tools_proceed_to_model_call(monkeypatch): + from mcp.types import Tool as MCPTool + + resolved = [MCPTool(name="get_links", description="List links", inputSchema={"type": "object"})] + _patch_resolved_tools(monkeypatch, resolved) + + response = _model_response() + aresponses_mock = AsyncMock(return_value=response) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + result = await responses_main_module.aresponses_api_with_mcp( + input="how many links do i have?", + model="gpt-4", + stream=False, + tools=[MCP_TOOL], + ) + + assert result is response + aresponses_mock.assert_called_once() + assert aresponses_mock.call_args.kwargs["tools"], "model call must carry the resolved tools" + + +@pytest.mark.asyncio +async def test_request_without_mcp_tools_is_unaffected(monkeypatch): + """Plain function-tool requests never hit the guard.""" + response = _model_response() + aresponses_mock = AsyncMock(return_value=response) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + result = await responses_main_module.aresponses_api_with_mcp( + input="hello", + model="gpt-4", + stream=False, + tools=[{"type": "function", "name": "my_fn", "parameters": {}}], + ) + + assert result is response + aresponses_mock.assert_called_once() diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index d2e593b4896..d6cefaed1b6 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -101,12 +101,15 @@ async def test_eager_creation_reraises_pre_stream_failure_as_http_error(monkeypa the stashed creation failure, so the proxy returns a real 4xx/5xx before any SSE bytes are written instead of an HTTP 200 with a broken stream. """ + from mcp.types import Tool as MCPTool + from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + resolved_tool = MCPTool(name="read_wiki_contents", description="read", inputSchema={"type": "object"}) monkeypatch.setattr( LiteLLM_Proxy_MCP_Handler, "_process_mcp_tools_without_openai_transform", - AsyncMock(return_value=([], {})), + AsyncMock(return_value=([resolved_tool], {"read_wiki_contents": "deepwiki"})), ) boom = litellm.BadRequestError( message="Previous response with id 'resp_bogus' not found.", From d181176fb7220d42ff38106bc63947e1710e889b Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 15:09:21 +1000 Subject: [PATCH 4/6] fix(responses): monotonic sequence number for terminal error event; drop orphaned tool events on batch failure Review feedback (Greptile on #32579): - The terminal error event was numbered sequence_number=1, out of order after tool-execution events. __anext__ now tracks the highest sequence_number that passed through the stream and the error event is numbered after it. - A batch tool-execution failure queued mcp_call.in_progress events that never received a terminal per-item event. Those queued events are now dropped; the terminal error event carries the failure instead. Co-Authored-By: Claude Fable 5 --- .../responses/mcp/mcp_streaming_iterator.py | 18 +++++++++++++++++- .../mcp/test_mcp_streaming_iterator.py | 10 ++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index dae7bcbd085..5759960fa13 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -313,6 +313,10 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._initial_creation_error: Optional[Exception] = None self._stream_error: Optional[Exception] = None self._error_event_emitted = False + # Highest sequence_number emitted so far; the terminal `error` event + # must be numbered after it to keep the stream monotonic for strict + # clients. + self._last_sequence_number = 0 def _extract_mcp_headers_from_params(self) -> None: """Extract MCP headers from original request params to pass to tool calls""" @@ -386,7 +390,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): status_code = getattr(err, "status_code", None) return ErrorEvent( type=ResponsesAPIStreamEvents.ERROR, - sequence_number=1, + sequence_number=self._last_sequence_number + 1, error=ErrorEventError( type="mcp_gateway_error", code=str(status_code) if status_code is not None else "internal_error", @@ -399,6 +403,13 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): return self async def __anext__(self) -> ResponsesAPIStreamingResponse: + chunk = await self._anext_impl() + sequence_number = getattr(chunk, "sequence_number", None) + if isinstance(sequence_number, int) and sequence_number > self._last_sequence_number: + self._last_sequence_number = sequence_number + return chunk + + async def _anext_impl(self) -> ResponsesAPIStreamingResponse: """ Phase-based streaming: 1. initial_response - Stream the first LLM response (includes response.created, response.in_progress, response.output_item.added) @@ -733,6 +744,11 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): traceback.print_exc() self.tool_results = [] + # Drop the queued per-tool events: emitting mcp_call.in_progress + # items that never receive a completed/failed terminal event is a + # protocol deviation. The terminal `error` event carries the + # failure instead. + self.tool_execution_events = [] # Remember the failure. Without this, the follow-up call is made # with function_call items but no function_call_output items and # the provider rejects it with "No tool output found for function diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index d6cefaed1b6..b1c4b2d2844 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -229,6 +229,9 @@ async def test_tool_execution_failure_emits_error_event_and_skips_follow_up(monk assert "mcp server exploded" in error_events[0].error.message # The doomed follow-up call was never made. aresponses_mock.assert_not_called() + # No orphaned per-tool events: a batch failure must not emit + # mcp_call.in_progress items that never receive a terminal event. + assert all(getattr(c, "type", None) != ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS for c in chunks) @pytest.mark.asyncio @@ -257,6 +260,13 @@ async def test_follow_up_failure_emits_error_event(monkeypatch): assert "No tool output found" in error_events[0].error.message # Tool-execution events were still streamed before the error surfaced. assert any(getattr(c, "type", None) == ResponsesAPIStreamEvents.MCP_CALL_COMPLETED for c in chunks) + # The terminal error event keeps sequence numbers monotonic for strict clients. + prior_sequence_numbers = [ + c.sequence_number + for c in chunks + if isinstance(getattr(c, "sequence_number", None), int) and c is not error_events[0] + ] + assert error_events[0].sequence_number > max(prior_sequence_numbers) @pytest.mark.asyncio From 092bc794320277cd428d1329d534ddbe564ed337 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 15:49:45 +1000 Subject: [PATCH 5/6] test: trim redundant MCP gateway tests - Drop test_initial_call_success_does_not_emit_error_event: the tool-call happy path (test_tool_call_happy_path_emits_no_error_event) already guards against false-positive error events and exercises more of the changed code (tool-exec + follow-up success paths). - Drop the stream=True parametrization on the zero-resolved-tools guard: the guard runs before the stream/non-stream branch in aresponses_api_with_mcp, so both cases hit identical code. Co-Authored-By: Claude Fable 5 --- .../mcp/test_mcp_empty_resolved_tools.py | 7 ++++--- .../mcp/test_mcp_streaming_iterator.py | 18 ------------------ 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py b/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py index 1a71b14475f..b2b8e23535f 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py +++ b/tests/test_litellm/responses/mcp/test_mcp_empty_resolved_tools.py @@ -40,8 +40,9 @@ def _model_response() -> ResponsesAPIResponse: @pytest.mark.asyncio -@pytest.mark.parametrize("stream", [False, True]) -async def test_zero_resolved_mcp_tools_raises_before_model_call(monkeypatch, stream): +async def test_zero_resolved_mcp_tools_raises_before_model_call(monkeypatch): + # The guard runs before the stream/non-stream branch in + # aresponses_api_with_mcp, so one case covers both. _patch_resolved_tools(monkeypatch, []) aresponses_mock = AsyncMock() monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) @@ -50,7 +51,7 @@ async def test_zero_resolved_mcp_tools_raises_before_model_call(monkeypatch, str await responses_main_module.aresponses_api_with_mcp( input="how many links do i have?", model="gpt-4", - stream=stream, + stream=False, tools=[MCP_TOOL], ) diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index b1c4b2d2844..97c86437b9d 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -130,24 +130,6 @@ async def test_eager_creation_reraises_pre_stream_failure_as_http_error(monkeypa assert "resp_bogus" in str(excinfo.value) -@pytest.mark.asyncio -async def test_initial_call_success_does_not_emit_error_event(monkeypatch): - """Happy path is unchanged: no error event, stream flows as before.""" - monkeypatch.setattr( - responses_main_module, - "aresponses", - AsyncMock(return_value=_text_only_stream("all good")), - ) - - iterator = _make_lazy_iterator() - chunks = [chunk async for chunk in iterator] - - assert all(getattr(c, "type", None) != ResponsesAPIStreamEvents.ERROR for c in chunks) - completed = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] - assert len(completed) == 1 - assert iterator._initial_creation_error is None - - import types from unittest.mock import MagicMock From 60412fc03d4aea30160eeb33481dc4d0f2205a04 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 9 Jul 2026 10:07:07 -0700 Subject: [PATCH 6/6] Remove deepwiki MCP server configuration Removed MCP server configuration for deepwiki. --- litellm/proxy/dev_config.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml index 8ad7da6d21a..65a4b8e7cbf 100644 --- a/litellm/proxy/dev_config.yaml +++ b/litellm/proxy/dev_config.yaml @@ -211,8 +211,3 @@ litellm_settings: sandbox_tool_name: e2b_sandbox callbacks: - code_interpreter_interception - -mcp_servers: - deepwiki: - url: "https://mcp.deepwiki.com/mcp" - transport: "http"