scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

refactor(responses): drop use_responses_api_bridge; fix PLR0915

Browse source 
- Only use_chat_completions_api and openai/chat_completions/ opt into the bridge
- Extract MCP gateway and file_search emulation dispatch to cut responses() size
- Update docs and tests

Made-with: Cursor
This commit is contained in:
Sameer Kankute 2026-04-13 18:22:17 +05:30
parent 0a8bf4ec9e
commit 2506ccb2bc
No known key found for this signature in database
3 changed files with 272 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/my-website/docs/response_api.md
View file

@ -1509,11 +1509,10 @@ curl http://localhost:4000/v1/responses \
If you're using an **OpenAI-compatible third-party provider** (e.g. llama.cpp, vLLM, LM Studio) via `openai/` prefix with a custom `api_base`, LiteLLM will normally forward `/responses` requests directly to that endpoint. If the provider only supports `/chat/completions`, the request will fail.
Use any of these to force the `/responses` → `/chat/completions` bridge:
Use either of these to force the `/responses` → `/chat/completions` bridge:
1. **`use_chat_completions_api: true`** (recommended) — makes it explicit that LiteLLM will call the provider’s chat-completions API.
1. **`use_chat_completions_api: true`** — makes it explicit that LiteLLM will call the provider’s chat-completions API.
2. **`openai/chat_completions/<model_name>`** — same pattern as `responses/` on chat completions: the model id encodes the routing choice.
3. **`use_responses_api_bridge: true`** — deprecated alias for `use_chat_completions_api` (kept for backward compatibility).
#### Python SDK Usage

379
litellm/responses/main.py
View file

@ -661,7 +661,7 @@ def _normalize_openai_chat_completions_responses_model(model: str) -> tuple[str,
def _pop_use_chat_completions_api_kw(kwargs: Dict[str, Any]) -> bool:
"""Pop bridge flags; True if either requests the chat-completions path."""
"""Pop use_chat_completions_api; True when the chat-completions bridge is requested."""
use_cc = kwargs.pop("use_chat_completions_api", None)
return bool(use_cc)
@ -728,6 +728,175 @@ def _apply_managed_file_id_mapping(
return input, tools
def _responses_try_dispatch_mcp_gateway(
*,
tools: Optional[Iterable[ToolParam]],
input: Union[str, ResponseInputParam],
model: str,
include: Optional[List[ResponseIncludable]],
instructions: Optional[str],
max_output_tokens: Optional[int],
prompt: Optional[PromptObject],
metadata: Optional[Dict[str, Any]],
parallel_tool_calls: Optional[bool],
previous_response_id: Optional[str],
reasoning: Optional[Reasoning],
store: Optional[bool],
background: Optional[bool],
stream: Optional[bool],
temperature: Optional[float],
text: Any,
tool_choice: Optional[ToolChoice],
top_p: Optional[float],
truncation: Optional[Literal["auto", "disabled"]],
user: Optional[str],
extra_headers: Optional[Dict[str, Any]],
extra_query: Optional[Dict[str, Any]],
extra_body: Optional[Dict[str, Any]],
timeout: Optional[Union[float, httpx.Timeout]],
custom_llm_provider: Optional[str],
kwargs: Dict[str, Any],
_is_async: bool,
) -> Optional[Any]:
"""Return a response when MCP gateway handles the call; otherwise None."""
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
LiteLLM_Proxy_MCP_Handler,
)
if not LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools):
return None
mcp_call_kwargs = {
"input": input,
"model": model,
"include": include,
"instructions": instructions,
"max_output_tokens": max_output_tokens,
"prompt": prompt,
"metadata": metadata,
"parallel_tool_calls": parallel_tool_calls,
"previous_response_id": previous_response_id,
"reasoning": reasoning,
"store": store,
"background": background,
"stream": stream,
"temperature": temperature,
"text": text,
"tool_choice": tool_choice,
"tools": tools,
"top_p": top_p,
"truncation": truncation,
"user": user,
"extra_headers": extra_headers,
"extra_query": extra_query,
"extra_body": extra_body,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
**kwargs,
}
if _is_async:
return aresponses_api_with_mcp(**mcp_call_kwargs)
return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs)
def _responses_try_dispatch_emulated_file_search(
*,
tools: Optional[Iterable[ToolParam]],
input: Union[str, ResponseInputParam],
model: str,
responses_api_provider_config: Optional[BaseResponsesAPIConfig],
use_chat_completions_api: bool,
include: Optional[List[ResponseIncludable]],
instructions: Optional[str],
max_output_tokens: Optional[int],
prompt: Optional[PromptObject],
metadata: Optional[Dict[str, Any]],
parallel_tool_calls: Optional[bool],
previous_response_id: Optional[str],
reasoning: Optional[Reasoning],
store: Optional[bool],
background: Optional[bool],
stream: Optional[bool],
temperature: Optional[float],
text: Any,
tool_choice: Optional[ToolChoice],
top_p: Optional[float],
truncation: Optional[Literal["auto", "disabled"]],
user: Optional[str],
service_tier: Optional[str],
safety_identifier: Optional[str],
text_format: Optional[Union[Type[BaseModel], dict]],
allowed_openai_params: Optional[List[str]],
extra_headers: Optional[Dict[str, Any]],
extra_query: Optional[Dict[str, Any]],
extra_body: Optional[Dict[str, Any]],
timeout: Optional[Union[float, httpx.Timeout]],
custom_llm_provider: Optional[str],
kwargs: Dict[str, Any],
_is_async: bool,
) -> Optional[Any]:
"""Return a response when emulated file_search handles the call; otherwise None."""
if not _has_file_search_tool(tools) or not (
responses_api_provider_config is None
or use_chat_completions_api is True
or not responses_api_provider_config.supports_native_file_search()
):
return None
from litellm.responses.file_search.emulated_handler import (
aresponses_with_emulated_file_search,
)
_internal_skip = {"litellm_call_id", "aresponses"}
emulated_kwargs = {
"include": include,
"instructions": instructions,
"max_output_tokens": max_output_tokens,
"prompt": prompt,
"metadata": metadata,
"parallel_tool_calls": parallel_tool_calls,
"previous_response_id": previous_response_id,
"reasoning": reasoning,
"store": store,
"background": background,
"stream": stream,
"temperature": temperature,
"text": text,
"tool_choice": tool_choice,
"top_p": top_p,
"truncation": truncation,
"user": user,
"service_tier": service_tier,
"safety_identifier": safety_identifier,
"text_format": text_format,
"allowed_openai_params": allowed_openai_params,
"extra_headers": extra_headers,
"extra_query": extra_query,
"extra_body": extra_body,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
**(
{
**(
{"use_chat_completions_api": True}
if use_chat_completions_api
else {}
),
**{k: v for k, v in kwargs.items() if k not in _internal_skip},
}
),
}
if _is_async:
return aresponses_with_emulated_file_search(
input=input, model=model, tools=tools, **emulated_kwargs
)
return run_async_function(
aresponses_with_emulated_file_search,
input=input,
model=model,
tools=tools,
**emulated_kwargs,
)
@client
def responses(
input: Union[str, ResponseInputParam],
@ -769,9 +938,6 @@ def responses(
Uses the synchronous HTTP handler to make requests.
"""
local_vars = locals()
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
LiteLLM_Proxy_MCP_Handler,
)
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
@ -841,38 +1007,37 @@ def responses(
#########################################################
# Native MCP Responses API
#########################################################
if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools):
mcp_call_kwargs = {
"input": input,
"model": model,
"include": include,
"instructions": instructions,
"max_output_tokens": max_output_tokens,
"prompt": prompt,
"metadata": metadata,
"parallel_tool_calls": parallel_tool_calls,
"previous_response_id": previous_response_id,
"reasoning": reasoning,
"store": store,
"background": background,
"stream": stream,
"temperature": temperature,
"text": text,
"tool_choice": tool_choice,
"tools": tools,
"top_p": top_p,
"truncation": truncation,
"user": user,
"extra_headers": extra_headers,
"extra_query": extra_query,
"extra_body": extra_body,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
**kwargs,
}
if _is_async:
return aresponses_api_with_mcp(**mcp_call_kwargs)
return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs)
_mcp_dispatch = _responses_try_dispatch_mcp_gateway(
tools=tools,
input=input,
model=model,
include=include,
instructions=instructions,
max_output_tokens=max_output_tokens,
prompt=prompt,
metadata=metadata,
parallel_tool_calls=parallel_tool_calls,
previous_response_id=previous_response_id,
reasoning=reasoning,
store=store,
background=background,
stream=stream,
temperature=temperature,
text=text,
tool_choice=tool_choice,
top_p=top_p,
truncation=truncation,
user=user,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
_is_async=_is_async,
)
if _mcp_dispatch is not None:
return _mcp_dispatch
# get provider config
responses_api_provider_config: Optional[BaseResponsesAPIConfig]
@ -902,61 +1067,43 @@ def responses(
)
)
if _has_file_search_tool(tools) and (
responses_api_provider_config is None
or use_chat_completions_api is True
or not responses_api_provider_config.supports_native_file_search()
):
from litellm.responses.file_search.emulated_handler import (
aresponses_with_emulated_file_search,
)
_internal_skip = {"litellm_call_id", "aresponses"}
emulated_kwargs = {
"include": include,
"instructions": instructions,
"max_output_tokens": max_output_tokens,
"prompt": prompt,
"metadata": metadata,
"parallel_tool_calls": parallel_tool_calls,
"previous_response_id": previous_response_id,
"reasoning": reasoning,
"store": store,
"background": background,
"stream": stream,
"temperature": temperature,
"text": text,
"tool_choice": tool_choice,
"top_p": top_p,
"truncation": truncation,
"user": user,
"service_tier": service_tier,
"safety_identifier": safety_identifier,
"text_format": text_format,
"allowed_openai_params": allowed_openai_params,
"extra_headers": extra_headers,
"extra_query": extra_query,
"extra_body": extra_body,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
**(
{"use_chat_completions_api": True}
if use_chat_completions_api
else {}
),
**{k: v for k, v in kwargs.items() if k not in _internal_skip},
}
if _is_async:
return aresponses_with_emulated_file_search(
input=input, model=model, tools=tools, **emulated_kwargs
)
return run_async_function(
aresponses_with_emulated_file_search,
input=input,
model=model,
tools=tools,
**emulated_kwargs,
)
_file_search_dispatch = _responses_try_dispatch_emulated_file_search(
tools=tools,
input=input,
model=model,
responses_api_provider_config=responses_api_provider_config,
use_chat_completions_api=use_chat_completions_api,
include=include,
instructions=instructions,
max_output_tokens=max_output_tokens,
prompt=prompt,
metadata=metadata,
parallel_tool_calls=parallel_tool_calls,
previous_response_id=previous_response_id,
reasoning=reasoning,
store=store,
background=background,
stream=stream,
temperature=temperature,
text=text,
tool_choice=tool_choice,
top_p=top_p,
truncation=truncation,
user=user,
service_tier=service_tier,
safety_identifier=safety_identifier,
text_format=text_format,
allowed_openai_params=allowed_openai_params,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
_is_async=_is_async,
)
if _file_search_dispatch is not None:
return _file_search_dispatch
if responses_api_provider_config is None or use_chat_completions_api is True:
return litellm_completion_transformation_handler.response_api_handler(
@ -1154,11 +1301,11 @@ def delete_responses(
raise ValueError("custom_llm_provider is required but passed as None")
# get provider config
responses_api_provider_config: Optional[
BaseResponsesAPIConfig
] = ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
responses_api_provider_config: Optional[BaseResponsesAPIConfig] = (
ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
)
)
if responses_api_provider_config is None:
@ -1335,11 +1482,11 @@ def get_responses(
raise ValueError("custom_llm_provider is required but passed as None")
# get provider config
responses_api_provider_config: Optional[
BaseResponsesAPIConfig
] = ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
responses_api_provider_config: Optional[BaseResponsesAPIConfig] = (
ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
)
)
if responses_api_provider_config is None:
@ -1493,11 +1640,11 @@ def list_input_items(
if custom_llm_provider is None:
raise ValueError("custom_llm_provider is required but passed as None")
responses_api_provider_config: Optional[
BaseResponsesAPIConfig
] = ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
responses_api_provider_config: Optional[BaseResponsesAPIConfig] = (
ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
)
)
if responses_api_provider_config is None:
@ -1652,11 +1799,11 @@ def cancel_responses(
raise ValueError("custom_llm_provider is required but passed as None")
# get provider config
responses_api_provider_config: Optional[
BaseResponsesAPIConfig
] = ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
responses_api_provider_config: Optional[BaseResponsesAPIConfig] = (
ProviderConfigManager.get_provider_responses_api_config(
model=None,
provider=custom_llm_provider,
)
)
if responses_api_provider_config is None:
@ -1840,11 +1987,11 @@ def compact_responses(
raise ValueError("custom_llm_provider is required but passed as None")
# get provider config
responses_api_provider_config: Optional[
BaseResponsesAPIConfig
] = ProviderConfigManager.get_provider_responses_api_config(
model=model,
provider=custom_llm_provider,
responses_api_provider_config: Optional[BaseResponsesAPIConfig] = (
ProviderConfigManager.get_provider_responses_api_config(
model=model,
provider=custom_llm_provider,
)
)
if responses_api_provider_config is None:

41
tests/test_litellm/responses/test_responses_api_bridge_flag.py
View file

@ -1,7 +1,6 @@
"""
Tests for forcing the /responses → /chat/completions bridge for `openai/` models
(via `use_chat_completions_api`, deprecated `use_responses_api_bridge`, or the
`openai/chat_completions/<model>` model id).
(via `use_chat_completions_api` or the `openai/chat_completions/<model>` model id).
Includes file_search emulation: the flag must be forwarded on inner aresponses
calls so routed requests do not hit a custom api_base /v1/responses endpoint.
@ -22,28 +21,6 @@ from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
class TestUseResponsesApiBridgeFlag:
"""Test that bridge opt-in forces the chat completions path."""
@patch(
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
)
@patch(
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
)
def test_bridge_used_when_flag_is_true(self, mock_get_config, mock_bridge_handler):
"""When use_responses_api_bridge=True (deprecated alias), the bridge runs."""
# Setup: provider config returns a non-None config (native support exists)
mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig()
mock_bridge_handler.return_value = MagicMock()
litellm.responses(
model="openai/my-custom-model",
input="Hello",
use_responses_api_bridge=True,
litellm_logging_obj=MagicMock(),
)
mock_bridge_handler.assert_called_once()
@patch(
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
)
@ -96,7 +73,7 @@ class TestUseResponsesApiBridgeFlag:
def test_native_forwarding_when_flag_absent(
self, mock_get_config, mock_native_handler
):
"""When use_responses_api_bridge is not set, openai/ models should use
"""When use_chat_completions_api is not set, openai/ models should use
native responses API forwarding (existing behavior)."""
mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig()
mock_native_handler.return_value = MagicMock()
@ -116,22 +93,19 @@ class TestUseResponsesApiBridgeFlag:
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
)
def test_flag_does_not_leak_into_kwargs(self, mock_get_config, mock_bridge_handler):
"""The use_responses_api_bridge flag should be popped from kwargs and not
passed through to the bridge handler."""
"""use_chat_completions_api should be popped and not passed to the bridge handler."""
mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig()
mock_bridge_handler.return_value = MagicMock()
litellm.responses(
model="openai/my-custom-model",
input="Hello",
use_responses_api_bridge=True,
use_chat_completions_api=True,
litellm_logging_obj=MagicMock(),
)
call_kwargs = mock_bridge_handler.call_args
# Bridge flags should not appear in the kwargs passed to the bridge handler
all_kwargs = call_kwargs.kwargs if call_kwargs.kwargs else {}
assert "use_responses_api_bridge" not in all_kwargs
assert "use_chat_completions_api" not in all_kwargs
@patch(
@ -163,7 +137,7 @@ class TestUseResponsesApiBridgeFlag:
async def test_bridge_flag_forwarded_to_file_search_emulation(
self, mock_get_config, mock_call_aresponses
):
"""When use_responses_api_bridge=True and file_search tool is present,
"""When use_chat_completions_api=True and file_search tool is present,
the flag should be forwarded to the inner aresponses call in the
file_search emulation path."""
# Setup: provider has native responses API support
@ -187,7 +161,7 @@ class TestUseResponsesApiBridgeFlag:
model="openai/my-custom-model",
input="Search for information",
tools=[{"type": "file_search"}],
use_responses_api_bridge=True,
use_chat_completions_api=True,
litellm_logging_obj=MagicMock(),
)
@ -257,7 +231,7 @@ class TestUseResponsesApiBridgeFlag:
"file_search": {"vector_store_ids": ["vs_123"]},
}
],
use_responses_api_bridge=True,
use_chat_completions_api=True,
api_base="http://localhost:8080/v1",
litellm_logging_obj=MagicMock(),
)
@ -268,7 +242,6 @@ class TestUseResponsesApiBridgeFlag:
)
for call in mock_bridge_handler.call_args_list:
all_kwargs = call.kwargs if call.kwargs else {}
assert "use_responses_api_bridge" not in all_kwargs
assert "use_chat_completions_api" not in all_kwargs
assert result is not None
assert result.id is not None