mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(mcp): answer 400 for MCP servers a route cannot run on Messages, Ollama and the Responses playground
/v1/messages requests with mcp_servers on a chat-adapter or Responses-bridge route now answer 400 naming each server, or drop it with a warning under drop_params, instead of being forwarded or silently ignored. Ollama generate no longer pastes a hosted MCP tool (headers included) into the prompt, and ollama_chat filters hosted MCP tools the same way. The Admin UI Responses playground now sends litellm_proxy/mcp references like the other endpoints.
This commit is contained in:
mateo-berri
parent
29efa23f78
commit
98cd008ba6
11 changed files with 367 additions and 73 deletions
|
|
@ -185,27 +185,71 @@ def _hosted_mcp_server_label(tool: object) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _drop_params_hint(dropped: str) -> str:
|
||||
return (
|
||||
f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n"
|
||||
"`litellm_settings:\n drop_params: true`\n"
|
||||
)
|
||||
|
||||
|
||||
def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError:
|
||||
return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400)
|
||||
|
||||
|
||||
def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]:
|
||||
server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None)
|
||||
if not server_labels:
|
||||
return tools
|
||||
named_servers: Final = ", ".join(f"`{label}`" for label in server_labels)
|
||||
if not drops_unsupported_params(drop_params):
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM '
|
||||
"run the MCP server for this model, register it on the LiteLLM MCP gateway and send "
|
||||
'`server_url: "litellm_proxy/mcp/<server name>"`. To drop hosted MCP tools instead, set '
|
||||
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
raise _unrunnable_mcp_error(
|
||||
f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM '
|
||||
"run the MCP server for this model, register it on the LiteLLM MCP gateway and send "
|
||||
'`server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("hosted MCP tools"),
|
||||
model=model,
|
||||
llm_provider=route,
|
||||
status_code=400,
|
||||
route=route,
|
||||
)
|
||||
verbose_logger.warning(HOSTED_MCP_DROP_WARNING, named_servers, model, route)
|
||||
return [tool for tool in tools if _hosted_mcp_server_label(tool) is None]
|
||||
|
||||
|
||||
MCP_SERVERS_DROP_WARNING: Final = (
|
||||
"Dropping mcp_servers %s for model=%s on %s (drop_params=True) because this route cannot run an MCP server."
|
||||
)
|
||||
|
||||
|
||||
def _mcp_server_name(server: object) -> str:
|
||||
match server:
|
||||
case {"name": str() as name}:
|
||||
return name
|
||||
case _:
|
||||
return "unnamed"
|
||||
|
||||
|
||||
def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]:
|
||||
if not mcp_servers:
|
||||
return ()
|
||||
if not isinstance(mcp_servers, list):
|
||||
return ("unnamed",)
|
||||
return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers)))
|
||||
|
||||
|
||||
def check_mcp_servers_droppable(mcp_servers: object, model: str, route: str, drop_params: bool) -> None:
|
||||
server_names: Final = _mcp_server_names(mcp_servers)
|
||||
if not server_names:
|
||||
return
|
||||
named_servers: Final = ", ".join(f"`{name}`" for name in server_names)
|
||||
if not drops_unsupported_params(drop_params):
|
||||
raise _unrunnable_mcp_error(
|
||||
f"{model} on {route} cannot run the MCP server {named_servers} from `mcp_servers`. To have LiteLLM run "
|
||||
'it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` tool with '
|
||||
'`server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("`mcp_servers`"),
|
||||
model=model,
|
||||
route=route,
|
||||
)
|
||||
verbose_logger.warning(MCP_SERVERS_DROP_WARNING, named_servers, model, route)
|
||||
|
||||
|
||||
_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"})
|
||||
_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object])
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.tool_fields import check_mcp_servers_droppable
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
)
|
||||
|
|
@ -52,7 +53,7 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
# Anthropic-only keys already mapped by the translator; strip on extra_kwargs re-merge.
|
||||
ANTHROPIC_ONLY_REQUEST_KEYS: Final[frozenset[str]] = frozenset({"output_config", "safeguards"})
|
||||
ANTHROPIC_ONLY_REQUEST_KEYS: Final[frozenset[str]] = frozenset({"mcp_servers", "output_config", "safeguards"})
|
||||
_REQUEST_KWARGS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
_AnthropicMessages: TypeAlias = "list[dict[str, object]]"
|
||||
|
|
@ -646,13 +647,18 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
request_data["output_config"] = extra_kwargs["output_config"]
|
||||
|
||||
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
|
||||
target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
|
||||
drop_params: Final = drops_unsupported_params(extra_kwargs.get("drop_params"))
|
||||
check_mcp_servers_droppable(
|
||||
extra_kwargs.get("mcp_servers"), model=model, route=target_provider or "this route", drop_params=drop_params
|
||||
)
|
||||
(
|
||||
openai_request,
|
||||
tool_name_mapping,
|
||||
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
|
||||
request_data,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
|
||||
custom_llm_provider=target_provider,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
if openai_request is None:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.tool_fields import check_mcp_servers_droppable
|
||||
from litellm.responses.utils import auto_reasoning_summary
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -144,6 +145,13 @@ def _build_responses_kwargs(
|
|||
"""
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
|
||||
target_provider: Final = forwarded_kwargs.get("custom_llm_provider")
|
||||
check_mcp_servers_droppable(
|
||||
forwarded_kwargs.get("mcp_servers"),
|
||||
model=model,
|
||||
route=target_provider if isinstance(target_provider, str) else "this route",
|
||||
drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")),
|
||||
)
|
||||
block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
|
||||
request_messages: Final[AnthropicRequestMessages] = (
|
||||
[
|
||||
|
|
@ -219,7 +227,7 @@ def _build_responses_kwargs(
|
|||
responses_kwargs["stream"] = True
|
||||
|
||||
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
|
||||
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
|
||||
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages", "mcp_servers"}
|
||||
for key, value in forwarded_kwargs.items():
|
||||
if key == "litellm_logging_obj" and value is not None:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
|
|||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from httpx._models import Headers, Response
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
extract_images_from_message,
|
||||
extract_reasoning_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
|
|
@ -39,6 +40,8 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class OllamaChatConfig(BaseConfig):
|
||||
"""
|
||||
|
|
@ -151,7 +154,13 @@ class OllamaChatConfig(BaseConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for param, value in non_default_params.items():
|
||||
request_params: Final = tool_params_without_hosted_mcp(
|
||||
_REQUEST_PARAMS.validate_python(non_default_params),
|
||||
model=model,
|
||||
route="ollama_chat",
|
||||
drop_params=drop_params,
|
||||
)
|
||||
for param, value in request_params.items():
|
||||
if param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["num_predict"] = value
|
||||
if param == "stream":
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ from litellm.litellm_core_utils.request_timeout_resolver import (
|
|||
get_configured_request_timeout,
|
||||
)
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
|
||||
from litellm.llms.base_llm.base_model_iterator import (
|
||||
|
|
@ -5955,14 +5956,20 @@ def completion(
|
|||
optional_params.pop("functions_unsupported_model")
|
||||
)
|
||||
request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
messages = function_call_prompt(
|
||||
messages=messages,
|
||||
functions=functions_unsupported_model,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
drop_params=litellm.drop_params is True
|
||||
or normalize_drop_params(request_kwargs.get("drop_params")) is True,
|
||||
prompt_drop_params: Final = (
|
||||
litellm.drop_params is True or normalize_drop_params(request_kwargs.get("drop_params")) is True
|
||||
)
|
||||
prompt_functions: Final = tools_without_hosted_mcp(
|
||||
functions_unsupported_model, model=model, route=custom_llm_provider, drop_params=prompt_drop_params
|
||||
)
|
||||
if prompt_functions:
|
||||
messages = function_call_prompt(
|
||||
messages=messages,
|
||||
functions=list(prompt_functions),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
drop_params=prompt_drop_params,
|
||||
)
|
||||
|
||||
# For logging - save the values of the litellm-specific params passed in
|
||||
requested_litellm_params: Final = get_litellm_params(
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ from unittest.mock import MagicMock, patch
|
|||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
# Anchor sys.path to this file's location — not the working-directory-relative
|
||||
# pattern Greptile flagged on PR #23706. Resolves correctly regardless of
|
||||
# where pytest is invoked from.
|
||||
|
|
@ -263,3 +265,56 @@ async def test_messages_layer_converted_stream_flag_still_gets_a_message_back(co
|
|||
)
|
||||
|
||||
assert response["content"][0]["text"] == "hi"
|
||||
|
||||
|
||||
_DEEPWIKI_SERVER: Final = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mcp_servers", "named"),
|
||||
[
|
||||
([_DEEPWIKI_SERVER], "`deepwiki`"),
|
||||
([_DEEPWIKI_SERVER, {"type": "url", "url": "https://mcp.example.com/mcp"}], "`deepwiki`, `unnamed`"),
|
||||
({"name": "deepwiki"}, "`unnamed`"),
|
||||
],
|
||||
ids=["named", "one-unnamed", "not-a-list"],
|
||||
)
|
||||
def test_mcp_servers_a_chat_route_cannot_run_answer_400_naming_each_server(
|
||||
monkeypatch: pytest.MonkeyPatch, mcp_servers: object, named: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as raised:
|
||||
_call_prepare(
|
||||
extra_kwargs={"mcp_servers": mcp_servers, "custom_llm_provider": "gemini"}, model="gemini-3.8-flash"
|
||||
)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert f"gemini-3.8-flash on gemini cannot run the MCP server {named} from `mcp_servers`" in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", ["request", "global"])
|
||||
def test_drop_params_drops_mcp_servers_instead_of_forwarding_them(
|
||||
monkeypatch: pytest.MonkeyPatch, drop_params: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", drop_params == "global")
|
||||
completion_kwargs, _ = _call_prepare(
|
||||
extra_kwargs={
|
||||
"mcp_servers": [_DEEPWIKI_SERVER],
|
||||
"custom_llm_provider": "gemini",
|
||||
**({"drop_params": True} if drop_params == "request" else {}),
|
||||
},
|
||||
model="gemini-3.8-flash",
|
||||
)
|
||||
|
||||
assert "mcp_servers" not in completion_kwargs
|
||||
assert completion_kwargs["messages"] == MESSAGES
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mcp_servers", [None, []], ids=["absent", "empty"])
|
||||
def test_no_mcp_servers_needs_no_drop_params(monkeypatch: pytest.MonkeyPatch, mcp_servers: object) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
completion_kwargs, _ = _call_prepare(
|
||||
extra_kwargs={"mcp_servers": mcp_servers, "custom_llm_provider": "gemini"}, model="gemini-3.8-flash"
|
||||
)
|
||||
|
||||
assert "mcp_servers" not in completion_kwargs
|
||||
|
|
|
|||
|
|
@ -739,3 +739,33 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
|
|||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
|
||||
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
|
||||
|
||||
|
||||
_DEEPWIKI_SERVER: Final = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"}
|
||||
|
||||
|
||||
def test_mcp_servers_the_responses_bridge_cannot_run_answer_400(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as rejected:
|
||||
_build_responses_kwargs(
|
||||
model="gpt-5.6",
|
||||
extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER]},
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
)
|
||||
|
||||
assert rejected.value.status_code == 400
|
||||
assert "gpt-5.6 on openai cannot run the MCP server `deepwiki` from `mcp_servers`" in rejected.value.message
|
||||
|
||||
|
||||
def test_drop_params_keeps_mcp_servers_off_the_responses_request(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
model="gpt-5.6",
|
||||
extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER], "drop_params": True},
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
)
|
||||
|
||||
assert "mcp_servers" not in responses_kwargs
|
||||
assert responses_kwargs["input"]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import sys
|
|||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import BaseModel
|
||||
|
||||
sys.path.insert(
|
||||
|
|
@ -1114,3 +1115,69 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str):
|
|||
)
|
||||
== "http://gateway.example.com/ollama/api/chat"
|
||||
)
|
||||
|
||||
|
||||
_DEEPWIKI_MCP_TOOL: Final = {
|
||||
"type": "mcp",
|
||||
"server_label": "deepwiki",
|
||||
"server_url": "https://mcp.deepwiki.com/mcp",
|
||||
"require_approval": "never",
|
||||
}
|
||||
_WEATHER_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}},
|
||||
}
|
||||
|
||||
|
||||
def _ollama_chat_route(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post("http://ollama.test:11434/api/chat").respond(
|
||||
json={
|
||||
"model": "qwen3:0.6b",
|
||||
"message": {"role": "assistant", "content": "Sunny"},
|
||||
"done": True,
|
||||
"prompt_eval_count": 5,
|
||||
"eval_count": 3,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_ollama_chat_answers_400_for_a_hosted_mcp_tool_it_cannot_run(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = _ollama_chat_route(respx_mock)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as raised:
|
||||
litellm.completion(
|
||||
model="ollama_chat/qwen3:0.6b",
|
||||
messages=[{"role": "user", "content": "weather in Paris?"}],
|
||||
tools=[_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL],
|
||||
api_base="http://ollama.test:11434",
|
||||
)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert "qwen3:0.6b on ollama_chat cannot run the hosted MCP tool `deepwiki`" in str(raised.value)
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "sent_tools"),
|
||||
[([_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], [_WEATHER_TOOL]), ([_DEEPWIKI_MCP_TOOL], None)],
|
||||
ids=["beside-a-function", "alone"],
|
||||
)
|
||||
def test_ollama_chat_drop_params_sends_only_the_tools_ollama_can_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[object], sent_tools: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = _ollama_chat_route(respx_mock)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="ollama_chat/qwen3:0.6b",
|
||||
messages=[{"role": "user", "content": "weather in Paris?"}],
|
||||
tools=tools,
|
||||
api_base="http://ollama.test:11434",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content).get("tools") == sent_tools
|
||||
assert response.choices[0].message.content == "Sunny"
|
||||
|
|
|
|||
|
|
@ -990,3 +990,71 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str):
|
|||
)
|
||||
== "http://gateway.example.com/ollama/api/generate"
|
||||
)
|
||||
|
||||
|
||||
_DEEPWIKI_MCP_TOOL: Final = {
|
||||
"type": "mcp",
|
||||
"server_label": "deepwiki",
|
||||
"server_url": "https://mcp.deepwiki.com/mcp",
|
||||
"require_approval": "never",
|
||||
}
|
||||
_WEATHER_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _ollama_generate_route(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post("http://ollama.test:11434/api/generate").respond(
|
||||
json={"model": "llama3.2", "response": "Sunny", "done": True, "prompt_eval_count": 5, "eval_count": 3}
|
||||
)
|
||||
|
||||
|
||||
def test_ollama_answers_400_for_a_hosted_mcp_tool_instead_of_pasting_it_into_the_prompt(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "add_function_to_prompt", False)
|
||||
route: Final = _ollama_generate_route(respx_mock)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as raised:
|
||||
litellm.completion(
|
||||
model="ollama/llama3.2",
|
||||
messages=[{"role": "user", "content": "weather in Paris?"}],
|
||||
tools=[_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL],
|
||||
api_base="http://ollama.test:11434",
|
||||
)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert "cannot run the hosted MCP tool `deepwiki`" in str(raised.value)
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "prompt_names_weather"),
|
||||
[([_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], True), ([_DEEPWIKI_MCP_TOOL], False)],
|
||||
ids=["beside-a-function", "alone"],
|
||||
)
|
||||
def test_ollama_drop_params_keeps_a_hosted_mcp_tool_out_of_the_prompt(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[object], prompt_names_weather: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "add_function_to_prompt", False)
|
||||
route: Final = _ollama_generate_route(respx_mock)
|
||||
|
||||
litellm.completion(
|
||||
model="ollama/llama3.2",
|
||||
messages=[{"role": "user", "content": "weather in Paris?"}],
|
||||
tools=tools,
|
||||
api_base="http://ollama.test:11434",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
prompt: Final = json.loads(route.calls.last.request.content)["prompt"]
|
||||
assert "deepwiki" not in prompt
|
||||
assert ("get_weather" in prompt) is prompt_names_weather
|
||||
assert prompt.startswith("### User:\nweather in Paris?")
|
||||
assert prompt_names_weather or prompt == "### User:\nweather in Paris?\n\n"
|
||||
|
|
|
|||
|
|
@ -207,6 +207,46 @@ describe("responses_api", () => {
|
|||
expect(onTimingData).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[["__all__"], [], [{ server_label: "litellm", server_url: "litellm_proxy/mcp" }]],
|
||||
[
|
||||
["server-1"],
|
||||
[{ server_id: "server-1", server_name: "my server" }],
|
||||
[{ server_label: "my server", server_url: "litellm_proxy/mcp/my server" }],
|
||||
],
|
||||
])(
|
||||
"sends MCP selection %j as gateway references the proxy runs for any model",
|
||||
async (selected, servers, expected) => {
|
||||
await makeOpenAIResponsesRequest(
|
||||
messages,
|
||||
mockUpdateTextUI,
|
||||
"gemini-pro",
|
||||
"test-token",
|
||||
undefined, // tags
|
||||
undefined, // signal
|
||||
undefined, // onReasoningContent
|
||||
undefined, // onTimingData
|
||||
undefined, // onUsageData
|
||||
undefined, // traceId
|
||||
undefined, // vector_store_ids
|
||||
undefined, // guardrails
|
||||
undefined, // policies
|
||||
selected,
|
||||
undefined, // previousResponseId
|
||||
undefined, // onResponseId
|
||||
undefined, // onMCPEvent
|
||||
undefined, // codeInterpreterEnabled
|
||||
undefined, // onCodeInterpreterResult
|
||||
undefined, // customBaseUrl
|
||||
servers as never,
|
||||
);
|
||||
|
||||
expect(mockResponsesCreate.mock.calls[0][0].tools).toEqual(
|
||||
expected.map((tool) => ({ type: "mcp", require_approval: "never", ...tool })),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("should report total latency in both streaming and non-streaming modes", async () => {
|
||||
const onTotalLatency = vi.fn();
|
||||
const callWithStreaming = (streamingEnabled: boolean) =>
|
||||
|
|
@ -475,14 +515,14 @@ describe("responses_api", () => {
|
|||
{
|
||||
type: "mcp",
|
||||
server_label: "Alpha",
|
||||
server_url: "https://backend.example.com/mcp/Alpha",
|
||||
server_url: "litellm_proxy/mcp/Alpha",
|
||||
require_approval: "never",
|
||||
allowed_tools: ["toolA"],
|
||||
},
|
||||
{
|
||||
type: "mcp",
|
||||
server_label: "Beta",
|
||||
server_url: "https://backend.example.com/mcp/Beta",
|
||||
server_url: "litellm_proxy/mcp/Beta",
|
||||
require_approval: "never",
|
||||
allowed_tools: ["toolB", "toolC"],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -3,11 +3,12 @@ import openai from "openai";
|
|||
import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses";
|
||||
import { MessageType } from "../chat_ui/types";
|
||||
import { TokenUsage } from "../chat_ui/ResponseMetrics";
|
||||
import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking";
|
||||
import { getGatewayBaseUrl } from "@/components/networking";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
|
||||
import { parseUsageCost } from "./usage_cost";
|
||||
import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers";
|
||||
import { buildMcpToolBlocks, type BuildMcpToolBlocksArgs } from "./mcp_tool_blocks";
|
||||
import type { MCPEvent } from "@/components/mcp_tools/types";
|
||||
import { MCPServer, MCPToolset } from "@/components/mcp_tools/types";
|
||||
import {
|
||||
|
|
@ -126,7 +127,6 @@ export async function makeOpenAIResponsesRequest(
|
|||
}
|
||||
|
||||
const proxyBaseUrl = customBaseUrl || getGatewayBaseUrl();
|
||||
const mcpBaseUrl = customBaseUrl || getProxyBaseUrl();
|
||||
const headers = buildPlaygroundHeaders(tags, customHeaders);
|
||||
|
||||
const client = new openai.OpenAI({
|
||||
|
|
@ -172,53 +172,13 @@ export async function makeOpenAIResponsesRequest(
|
|||
});
|
||||
|
||||
// Build tools array
|
||||
const tools: any[] = [];
|
||||
|
||||
// Add MCP servers if selected
|
||||
if (selectedMCPServers && selectedMCPServers.length > 0) {
|
||||
if (selectedMCPServers.includes("__all__")) {
|
||||
// All MCP Servers selected
|
||||
const allServersTool = {
|
||||
type: "mcp",
|
||||
server_label: "litellm",
|
||||
server_url: `${mcpBaseUrl}/mcp`,
|
||||
require_approval: "never",
|
||||
};
|
||||
tools.push(allServersTool);
|
||||
} else {
|
||||
// Individual servers/toolsets selected - create one entry per item
|
||||
selectedMCPServers.forEach((serverId) => {
|
||||
if (serverId.startsWith("toolset:")) {
|
||||
// Toolset: same /{name}/mcp pattern as individual servers
|
||||
const toolsetId = serverId.slice("toolset:".length);
|
||||
const toolset = mcpToolsets?.find((t) => t.toolset_id === toolsetId);
|
||||
const toolsetName = toolset?.toolset_name || toolsetId;
|
||||
const toolsetTool = {
|
||||
type: "mcp",
|
||||
server_label: toolsetName,
|
||||
server_url: `${mcpBaseUrl}/mcp/${encodeURIComponent(toolsetName)}`,
|
||||
require_approval: "never",
|
||||
};
|
||||
tools.push(toolsetTool);
|
||||
} else {
|
||||
const server = mcpServers?.find((s) => s.server_id === serverId);
|
||||
// Use server_name for both routing and labelling. server_name is the
|
||||
// unique registered identifier; aliases can collide across servers.
|
||||
const routeName = server?.server_name || serverId;
|
||||
const allowedTools = mcpServerToolRestrictions?.[serverId] || [];
|
||||
|
||||
const serverTool = {
|
||||
type: "mcp",
|
||||
server_label: routeName, // unique per request — collisions cause silent tool-routing failures
|
||||
server_url: `${mcpBaseUrl}/mcp/${encodeURIComponent(routeName)}`,
|
||||
require_approval: "never",
|
||||
...(allowedTools.length > 0 ? { allowed_tools: allowedTools } : {}),
|
||||
};
|
||||
tools.push(serverTool);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
const mcpSelection: BuildMcpToolBlocksArgs = {
|
||||
selectedMCPServers,
|
||||
mcpServers,
|
||||
mcpToolsets,
|
||||
mcpServerToolRestrictions,
|
||||
};
|
||||
const tools: any[] = buildMcpToolBlocks(mcpSelection);
|
||||
|
||||
// Add code_interpreter tool if enabled (OpenAI auto-creates container)
|
||||
if (codeInterpreterEnabled) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue