From 2506ccb2bc5d0ab3b8a9c5752c61c12a960b74c1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 13 Apr 2026 18:22:17 +0530 Subject: [PATCH] refactor(responses): drop use_responses_api_bridge; fix PLR0915 - Only use_chat_completions_api and openai/chat_completions/ opt into the bridge - Extract MCP gateway and file_search emulation dispatch to cut responses() size - Update docs and tests Made-with: Cursor --- docs/my-website/docs/response_api.md | 5 +- litellm/responses/main.py | 379 ++++++++++++------ .../test_responses_api_bridge_flag.py | 41 +- 3 files changed, 272 insertions(+), 153 deletions(-) diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 94f5c3e52fa..20bb6d50d97 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1509,11 +1509,10 @@ curl http://localhost:4000/v1/responses \ If you're using an **OpenAI-compatible third-party provider** (e.g. llama.cpp, vLLM, LM Studio) via `openai/` prefix with a custom `api_base`, LiteLLM will normally forward `/responses` requests directly to that endpoint. If the provider only supports `/chat/completions`, the request will fail. -Use any of these to force the `/responses` → `/chat/completions` bridge: +Use either of these to force the `/responses` → `/chat/completions` bridge: -1. **`use_chat_completions_api: true`** (recommended) — makes it explicit that LiteLLM will call the provider’s chat-completions API. +1. **`use_chat_completions_api: true`** — makes it explicit that LiteLLM will call the provider’s chat-completions API. 2. **`openai/chat_completions/`** — same pattern as `responses/` on chat completions: the model id encodes the routing choice. -3. **`use_responses_api_bridge: true`** — deprecated alias for `use_chat_completions_api` (kept for backward compatibility). #### Python SDK Usage diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 91e173a7a84..edd936e7344 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -661,7 +661,7 @@ def _normalize_openai_chat_completions_responses_model(model: str) -> tuple[str, def _pop_use_chat_completions_api_kw(kwargs: Dict[str, Any]) -> bool: - """Pop bridge flags; True if either requests the chat-completions path.""" + """Pop use_chat_completions_api; True when the chat-completions bridge is requested.""" use_cc = kwargs.pop("use_chat_completions_api", None) return bool(use_cc) @@ -728,6 +728,175 @@ def _apply_managed_file_id_mapping( return input, tools +def _responses_try_dispatch_mcp_gateway( + *, + tools: Optional[Iterable[ToolParam]], + input: Union[str, ResponseInputParam], + model: str, + include: Optional[List[ResponseIncludable]], + instructions: Optional[str], + max_output_tokens: Optional[int], + prompt: Optional[PromptObject], + metadata: Optional[Dict[str, Any]], + parallel_tool_calls: Optional[bool], + previous_response_id: Optional[str], + reasoning: Optional[Reasoning], + store: Optional[bool], + background: Optional[bool], + stream: Optional[bool], + temperature: Optional[float], + text: Any, + tool_choice: Optional[ToolChoice], + top_p: Optional[float], + truncation: Optional[Literal["auto", "disabled"]], + user: Optional[str], + extra_headers: Optional[Dict[str, Any]], + extra_query: Optional[Dict[str, Any]], + extra_body: Optional[Dict[str, Any]], + timeout: Optional[Union[float, httpx.Timeout]], + custom_llm_provider: Optional[str], + kwargs: Dict[str, Any], + _is_async: bool, +) -> Optional[Any]: + """Return a response when MCP gateway handles the call; otherwise None.""" + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + + if not LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools): + return None + mcp_call_kwargs = { + "input": input, + "model": model, + "include": include, + "instructions": instructions, + "max_output_tokens": max_output_tokens, + "prompt": prompt, + "metadata": metadata, + "parallel_tool_calls": parallel_tool_calls, + "previous_response_id": previous_response_id, + "reasoning": reasoning, + "store": store, + "background": background, + "stream": stream, + "temperature": temperature, + "text": text, + "tool_choice": tool_choice, + "tools": tools, + "top_p": top_p, + "truncation": truncation, + "user": user, + "extra_headers": extra_headers, + "extra_query": extra_query, + "extra_body": extra_body, + "timeout": timeout, + "custom_llm_provider": custom_llm_provider, + **kwargs, + } + if _is_async: + return aresponses_api_with_mcp(**mcp_call_kwargs) + return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs) + + +def _responses_try_dispatch_emulated_file_search( + *, + tools: Optional[Iterable[ToolParam]], + input: Union[str, ResponseInputParam], + model: str, + responses_api_provider_config: Optional[BaseResponsesAPIConfig], + use_chat_completions_api: bool, + include: Optional[List[ResponseIncludable]], + instructions: Optional[str], + max_output_tokens: Optional[int], + prompt: Optional[PromptObject], + metadata: Optional[Dict[str, Any]], + parallel_tool_calls: Optional[bool], + previous_response_id: Optional[str], + reasoning: Optional[Reasoning], + store: Optional[bool], + background: Optional[bool], + stream: Optional[bool], + temperature: Optional[float], + text: Any, + tool_choice: Optional[ToolChoice], + top_p: Optional[float], + truncation: Optional[Literal["auto", "disabled"]], + user: Optional[str], + service_tier: Optional[str], + safety_identifier: Optional[str], + text_format: Optional[Union[Type[BaseModel], dict]], + allowed_openai_params: Optional[List[str]], + extra_headers: Optional[Dict[str, Any]], + extra_query: Optional[Dict[str, Any]], + extra_body: Optional[Dict[str, Any]], + timeout: Optional[Union[float, httpx.Timeout]], + custom_llm_provider: Optional[str], + kwargs: Dict[str, Any], + _is_async: bool, +) -> Optional[Any]: + """Return a response when emulated file_search handles the call; otherwise None.""" + if not _has_file_search_tool(tools) or not ( + responses_api_provider_config is None + or use_chat_completions_api is True + or not responses_api_provider_config.supports_native_file_search() + ): + return None + from litellm.responses.file_search.emulated_handler import ( + aresponses_with_emulated_file_search, + ) + + _internal_skip = {"litellm_call_id", "aresponses"} + emulated_kwargs = { + "include": include, + "instructions": instructions, + "max_output_tokens": max_output_tokens, + "prompt": prompt, + "metadata": metadata, + "parallel_tool_calls": parallel_tool_calls, + "previous_response_id": previous_response_id, + "reasoning": reasoning, + "store": store, + "background": background, + "stream": stream, + "temperature": temperature, + "text": text, + "tool_choice": tool_choice, + "top_p": top_p, + "truncation": truncation, + "user": user, + "service_tier": service_tier, + "safety_identifier": safety_identifier, + "text_format": text_format, + "allowed_openai_params": allowed_openai_params, + "extra_headers": extra_headers, + "extra_query": extra_query, + "extra_body": extra_body, + "timeout": timeout, + "custom_llm_provider": custom_llm_provider, + **( + { + **( + {"use_chat_completions_api": True} + if use_chat_completions_api + else {} + ), + **{k: v for k, v in kwargs.items() if k not in _internal_skip}, + } + ), + } + if _is_async: + return aresponses_with_emulated_file_search( + input=input, model=model, tools=tools, **emulated_kwargs + ) + return run_async_function( + aresponses_with_emulated_file_search, + input=input, + model=model, + tools=tools, + **emulated_kwargs, + ) + + @client def responses( input: Union[str, ResponseInputParam], @@ -769,9 +938,6 @@ def responses( Uses the synchronous HTTP handler to make requests. """ local_vars = locals() - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) try: litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore @@ -841,38 +1007,37 @@ def responses( ######################################################### # Native MCP Responses API ######################################################### - if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools): - mcp_call_kwargs = { - "input": input, - "model": model, - "include": include, - "instructions": instructions, - "max_output_tokens": max_output_tokens, - "prompt": prompt, - "metadata": metadata, - "parallel_tool_calls": parallel_tool_calls, - "previous_response_id": previous_response_id, - "reasoning": reasoning, - "store": store, - "background": background, - "stream": stream, - "temperature": temperature, - "text": text, - "tool_choice": tool_choice, - "tools": tools, - "top_p": top_p, - "truncation": truncation, - "user": user, - "extra_headers": extra_headers, - "extra_query": extra_query, - "extra_body": extra_body, - "timeout": timeout, - "custom_llm_provider": custom_llm_provider, - **kwargs, - } - if _is_async: - return aresponses_api_with_mcp(**mcp_call_kwargs) - return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs) + _mcp_dispatch = _responses_try_dispatch_mcp_gateway( + tools=tools, + input=input, + model=model, + include=include, + instructions=instructions, + max_output_tokens=max_output_tokens, + prompt=prompt, + metadata=metadata, + parallel_tool_calls=parallel_tool_calls, + previous_response_id=previous_response_id, + reasoning=reasoning, + store=store, + background=background, + stream=stream, + temperature=temperature, + text=text, + tool_choice=tool_choice, + top_p=top_p, + truncation=truncation, + user=user, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + _is_async=_is_async, + ) + if _mcp_dispatch is not None: + return _mcp_dispatch # get provider config responses_api_provider_config: Optional[BaseResponsesAPIConfig] @@ -902,61 +1067,43 @@ def responses( ) ) - if _has_file_search_tool(tools) and ( - responses_api_provider_config is None - or use_chat_completions_api is True - or not responses_api_provider_config.supports_native_file_search() - ): - from litellm.responses.file_search.emulated_handler import ( - aresponses_with_emulated_file_search, - ) - - _internal_skip = {"litellm_call_id", "aresponses"} - emulated_kwargs = { - "include": include, - "instructions": instructions, - "max_output_tokens": max_output_tokens, - "prompt": prompt, - "metadata": metadata, - "parallel_tool_calls": parallel_tool_calls, - "previous_response_id": previous_response_id, - "reasoning": reasoning, - "store": store, - "background": background, - "stream": stream, - "temperature": temperature, - "text": text, - "tool_choice": tool_choice, - "top_p": top_p, - "truncation": truncation, - "user": user, - "service_tier": service_tier, - "safety_identifier": safety_identifier, - "text_format": text_format, - "allowed_openai_params": allowed_openai_params, - "extra_headers": extra_headers, - "extra_query": extra_query, - "extra_body": extra_body, - "timeout": timeout, - "custom_llm_provider": custom_llm_provider, - **( - {"use_chat_completions_api": True} - if use_chat_completions_api - else {} - ), - **{k: v for k, v in kwargs.items() if k not in _internal_skip}, - } - if _is_async: - return aresponses_with_emulated_file_search( - input=input, model=model, tools=tools, **emulated_kwargs - ) - return run_async_function( - aresponses_with_emulated_file_search, - input=input, - model=model, - tools=tools, - **emulated_kwargs, - ) + _file_search_dispatch = _responses_try_dispatch_emulated_file_search( + tools=tools, + input=input, + model=model, + responses_api_provider_config=responses_api_provider_config, + use_chat_completions_api=use_chat_completions_api, + include=include, + instructions=instructions, + max_output_tokens=max_output_tokens, + prompt=prompt, + metadata=metadata, + parallel_tool_calls=parallel_tool_calls, + previous_response_id=previous_response_id, + reasoning=reasoning, + store=store, + background=background, + stream=stream, + temperature=temperature, + text=text, + tool_choice=tool_choice, + top_p=top_p, + truncation=truncation, + user=user, + service_tier=service_tier, + safety_identifier=safety_identifier, + text_format=text_format, + allowed_openai_params=allowed_openai_params, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + _is_async=_is_async, + ) + if _file_search_dispatch is not None: + return _file_search_dispatch if responses_api_provider_config is None or use_chat_completions_api is True: return litellm_completion_transformation_handler.response_api_handler( @@ -1154,11 +1301,11 @@ def delete_responses( raise ValueError("custom_llm_provider is required but passed as None") # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=custom_llm_provider, + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=custom_llm_provider, + ) ) if responses_api_provider_config is None: @@ -1335,11 +1482,11 @@ def get_responses( raise ValueError("custom_llm_provider is required but passed as None") # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=custom_llm_provider, + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=custom_llm_provider, + ) ) if responses_api_provider_config is None: @@ -1493,11 +1640,11 @@ def list_input_items( if custom_llm_provider is None: raise ValueError("custom_llm_provider is required but passed as None") - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=custom_llm_provider, + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=custom_llm_provider, + ) ) if responses_api_provider_config is None: @@ -1652,11 +1799,11 @@ def cancel_responses( raise ValueError("custom_llm_provider is required but passed as None") # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=None, - provider=custom_llm_provider, + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=None, + provider=custom_llm_provider, + ) ) if responses_api_provider_config is None: @@ -1840,11 +1987,11 @@ def compact_responses( raise ValueError("custom_llm_provider is required but passed as None") # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=model, - provider=custom_llm_provider, + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=custom_llm_provider, + ) ) if responses_api_provider_config is None: diff --git a/tests/test_litellm/responses/test_responses_api_bridge_flag.py b/tests/test_litellm/responses/test_responses_api_bridge_flag.py index e635e125605..463af6562f1 100644 --- a/tests/test_litellm/responses/test_responses_api_bridge_flag.py +++ b/tests/test_litellm/responses/test_responses_api_bridge_flag.py @@ -1,7 +1,6 @@ """ Tests for forcing the /responses → /chat/completions bridge for `openai/` models -(via `use_chat_completions_api`, deprecated `use_responses_api_bridge`, or the -`openai/chat_completions/` model id). +(via `use_chat_completions_api` or the `openai/chat_completions/` model id). Includes file_search emulation: the flag must be forwarded on inner aresponses calls so routed requests do not hit a custom api_base /v1/responses endpoint. @@ -22,28 +21,6 @@ from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse class TestUseResponsesApiBridgeFlag: """Test that bridge opt-in forces the chat completions path.""" - @patch( - "litellm.responses.main.litellm_completion_transformation_handler.response_api_handler" - ) - @patch( - "litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config" - ) - def test_bridge_used_when_flag_is_true(self, mock_get_config, mock_bridge_handler): - """When use_responses_api_bridge=True (deprecated alias), the bridge runs.""" - # Setup: provider config returns a non-None config (native support exists) - mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() - - mock_bridge_handler.return_value = MagicMock() - - litellm.responses( - model="openai/my-custom-model", - input="Hello", - use_responses_api_bridge=True, - litellm_logging_obj=MagicMock(), - ) - - mock_bridge_handler.assert_called_once() - @patch( "litellm.responses.main.litellm_completion_transformation_handler.response_api_handler" ) @@ -96,7 +73,7 @@ class TestUseResponsesApiBridgeFlag: def test_native_forwarding_when_flag_absent( self, mock_get_config, mock_native_handler ): - """When use_responses_api_bridge is not set, openai/ models should use + """When use_chat_completions_api is not set, openai/ models should use native responses API forwarding (existing behavior).""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() mock_native_handler.return_value = MagicMock() @@ -116,22 +93,19 @@ class TestUseResponsesApiBridgeFlag: "litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config" ) def test_flag_does_not_leak_into_kwargs(self, mock_get_config, mock_bridge_handler): - """The use_responses_api_bridge flag should be popped from kwargs and not - passed through to the bridge handler.""" + """use_chat_completions_api should be popped and not passed to the bridge handler.""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() mock_bridge_handler.return_value = MagicMock() litellm.responses( model="openai/my-custom-model", input="Hello", - use_responses_api_bridge=True, + use_chat_completions_api=True, litellm_logging_obj=MagicMock(), ) call_kwargs = mock_bridge_handler.call_args - # Bridge flags should not appear in the kwargs passed to the bridge handler all_kwargs = call_kwargs.kwargs if call_kwargs.kwargs else {} - assert "use_responses_api_bridge" not in all_kwargs assert "use_chat_completions_api" not in all_kwargs @patch( @@ -163,7 +137,7 @@ class TestUseResponsesApiBridgeFlag: async def test_bridge_flag_forwarded_to_file_search_emulation( self, mock_get_config, mock_call_aresponses ): - """When use_responses_api_bridge=True and file_search tool is present, + """When use_chat_completions_api=True and file_search tool is present, the flag should be forwarded to the inner aresponses call in the file_search emulation path.""" # Setup: provider has native responses API support @@ -187,7 +161,7 @@ class TestUseResponsesApiBridgeFlag: model="openai/my-custom-model", input="Search for information", tools=[{"type": "file_search"}], - use_responses_api_bridge=True, + use_chat_completions_api=True, litellm_logging_obj=MagicMock(), ) @@ -257,7 +231,7 @@ class TestUseResponsesApiBridgeFlag: "file_search": {"vector_store_ids": ["vs_123"]}, } ], - use_responses_api_bridge=True, + use_chat_completions_api=True, api_base="http://localhost:8080/v1", litellm_logging_obj=MagicMock(), ) @@ -268,7 +242,6 @@ class TestUseResponsesApiBridgeFlag: ) for call in mock_bridge_handler.call_args_list: all_kwargs = call.kwargs if call.kwargs else {} - assert "use_responses_api_bridge" not in all_kwargs assert "use_chat_completions_api" not in all_kwargs assert result is not None assert result.id is not None