diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d6ebc44ac52..14fe4b58ed6 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -326,8 +326,13 @@ async def aresponses_api_with_mcp( ) if tool_results: + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( - response=response, tool_results=tool_results, original_input=input + response=response, + tool_results=tool_results, + original_input=input, + preserve_reasoning=persistence_disabled, ) # Prepare parameters for follow-up call (restores original stream setting) @@ -346,7 +351,7 @@ async def aresponses_api_with_mcp( follow_up_input=follow_up_input, model=model, all_tools=all_tools, - response_id=response.id, + response_id=None if persistence_disabled else response.id, **follow_up_call_params, ) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 197d0c02ba8..682e50e4af3 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -963,11 +963,30 @@ class LiteLLM_Proxy_MCP_Handler: return follow_up_messages + @staticmethod + def _is_persistence_disabled(call_params: Mapping[str, object]) -> bool: + """Whether the caller opted out of server-side response persistence (store=false). + + Zero data retention callers send store=false, so the provider never persisted the + first response and previous_response_id cannot be used to link the follow-up call. + """ + return call_params.get("store") is False + + @staticmethod + def _extract_reasoning_items(response: ResponsesAPIResponse) -> tuple[Mapping[str, object], ...]: + """Reasoning output items, kept whole so reasoning.encrypted_content survives replay.""" + normalized: Final = tuple( + output_item if isinstance(output_item, dict) else output_item.model_dump(exclude_none=True) + for output_item in response.output + ) + return tuple(item for item in normalized if item.get("type") == "reasoning") + @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, tool_results: Sequence[Mapping[str, object]], original_input: str | ResponseInputParam | None = None, + preserve_reasoning: bool = False, ) -> list[object]: """Create follow-up input with tool results in proper format.""" follow_up_input: Final[list[object]] = [] @@ -1025,6 +1044,9 @@ class LiteLLM_Proxy_MCP_Handler: } ) + if preserve_reasoning: + follow_up_input.extend(LiteLLM_Proxy_MCP_Handler._extract_reasoning_items(response)) + # Add function calls (these can come directly after user message for LLM) for function_call in function_calls: follow_up_input.append(function_call) @@ -1046,10 +1068,14 @@ class LiteLLM_Proxy_MCP_Handler: follow_up_input: list[Any], model: str, all_tools: Sequence[ResponsesToolParam] | None, - response_id: str, + response_id: str | None, **call_params: Any, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: - """Make follow-up response API call with tool results.""" + """Make follow-up response API call with tool results. + + response_id is None for stateless (store=false) requests, where the whole prior + turn is replayed in follow_up_input instead of linked by previous_response_id. + """ return await aresponses( input=follow_up_input, model=model, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 8f5dc926c68..83322b8d837 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -781,10 +781,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): try: # Create follow-up input if self.collected_response is not None: + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled( + self.original_request_params + ) + follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( response=self.collected_response, tool_results=self.tool_results, original_input=self.original_request_params.get("input"), + preserve_reasoning=persistence_disabled, ) # Make follow-up call with streaming @@ -795,6 +800,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): "stream": True, } ) + if persistence_disabled: + follow_up_params.pop("previous_response_id", None) else: return # Remove tool_choice to avoid forcing more tool calls diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index b33ed3bb581..8ebea685d5a 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -9,10 +9,13 @@ from fastapi import HTTPException import importlib from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.responses import main as responses_main +from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from typing import Any, cast +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse from litellm.types.responses.main import OutputFunctionToolCall @@ -719,3 +722,148 @@ def test_extract_tool_call_details_still_prefers_openai_arguments(): assert name == "get_weather" assert call_id == "call_123" assert arguments == '{"city": "Paris"}' + + +def _response_with_reasoning_and_tool_call() -> Any: + """A first-turn response as a reasoning model returns it: reasoning item, then a function call.""" + return ResponsesAPIResponse( + id="resp_first", + created_at=1234567890, + model="gpt-5", + object="response", + status="completed", + output=[ + { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "encrypted_content": "gAAAAA-opaque-blob", + }, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + "status": "completed", + }, + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + +def test_create_follow_up_input_preserves_reasoning_when_stateless(): + """ + Regression test (LIT-5427): a store=false follow-up has to replay the reasoning + item, including reasoning.encrypted_content, since the provider kept no state. + """ + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_reasoning_and_tool_call(), + tool_results=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}], + original_input="hi", + preserve_reasoning=True, + ) + + assert follow_up[1] == { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "encrypted_content": "gAAAAA-opaque-blob", + } + assert follow_up[2] == { + "type": "function_call", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + } + assert follow_up[3] == { + "type": "function_call_output", + "call_id": "call-1", + "output": "done", + } + + +def test_create_follow_up_input_omits_reasoning_when_stateful(): + """With store=true the provider still holds the reasoning item, so don't resend it.""" + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_reasoning_and_tool_call(), + tool_results=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}], + original_input="hi", + ) + + assert not [item for item in follow_up if isinstance(item, dict) and item.get("type") == "reasoning"] + + +@pytest.mark.parametrize( + "call_params, expected", + [ + ({"store": False}, True), + ({"store": True}, False), + ({"store": None}, False), + ({}, False), + ], +) +def test_is_persistence_disabled(call_params: dict[str, Any], expected: bool): + assert LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) is expected + + +@pytest.mark.parametrize( + "store, expected_previous_response_id", + [(False, None), (True, "resp_first")], +) +@pytest.mark.asyncio +async def test_mcp_follow_up_call_is_stateless_when_store_is_false( + monkeypatch: pytest.MonkeyPatch, store: bool, expected_previous_response_id: str | None +): + """ + Regression test (LIT-5427): linking the MCP follow-up call with + previous_response_id fails for zero data retention callers, because store=false + means the first response was never persisted. + """ + captured_calls: list[dict[str, Any]] = [] + first_response = _response_with_reasoning_and_tool_call() + + async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse: + captured_calls.append(kwargs) + return first_response if len(captured_calls) == 1 else ResponsesAPIResponse( + id="resp_follow_up", + created_at=1234567891, + model="gpt-5", + object="response", + status="completed", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + async def fake_process(**kwargs: Any) -> tuple[list[Any], dict[str, str]]: + return ([], {"foo": "litellm_proxy"}) + + async def fake_execute(**kwargs: Any) -> list[dict[str, Any]]: + return [{"tool_call_id": "call-1", "name": "foo", "result": "done"}] + + monkeypatch.setattr(responses_main, "aresponses", fake_aresponses) + monkeypatch.setattr(mcp_handler_module, "aresponses", fake_aresponses) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, "_process_mcp_tools_without_openai_transform", staticmethod(fake_process) + ) + monkeypatch.setattr(LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls", staticmethod(fake_execute)) + + await responses_main.aresponses_api_with_mcp( + input="hi", + model="gpt-5", + tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], + store=store, + ) + + assert len(captured_calls) == 2 + follow_up_call = captured_calls[1] + assert follow_up_call["previous_response_id"] == expected_previous_response_id + + reasoning_items = [ + item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning" + ] + assert bool(reasoning_items) is (store is False) diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index aacd614abb9..ac0c5ef6392 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -258,3 +258,81 @@ async def test_initial_call_failure_is_stashed_for_eager_reraise(monkeypatch): assert iterator._initial_creation_error is not None assert "initial boom" in str(iterator._initial_creation_error) + + +def _reasoning_item(encrypted_content: str): + return {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content} + + +@pytest.mark.asyncio +async def test_streaming_follow_up_is_stateless_when_store_is_false(monkeypatch): + """ + Regression test (LIT-5427): with store=false the provider persisted nothing, so the + streaming follow-up must drop previous_response_id and replay the reasoning item + (carrying reasoning.encrypted_content) instead of pointing at a response id. + """ + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(side_effect=[_text_only_stream("done")]) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_reasoning_item("gAAAAA-opaque-blob"), _function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + "store": False, + "previous_response_id": "resp_prev", + }, + ) + + _ = [chunk async for chunk in iterator] + + assert aresponses_mock.call_count == 1 + follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs + assert "previous_response_id" not in follow_up_kwargs + assert _reasoning_item("gAAAAA-opaque-blob") in follow_up_kwargs["input"] + + +@pytest.mark.asyncio +async def test_streaming_follow_up_keeps_previous_response_id_when_stored(monkeypatch): + """The stateful default is unchanged: previous_response_id still links the follow-up.""" + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(side_effect=[_text_only_stream("done")]) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_reasoning_item("gAAAAA-opaque-blob"), _function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + "previous_response_id": "resp_prev", + }, + ) + + _ = [chunk async for chunk in iterator] + + follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs + assert follow_up_kwargs["previous_response_id"] == "resp_prev" + assert not [item for item in follow_up_kwargs["input"] if item.get("type") == "reasoning"]