From 84a0d2950d949fddd3d07433aebb464d52128d39 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:30:41 -0800 Subject: [PATCH 1/3] fix(core): move drops_unsupported_params into core_helpers on lts/1.108.x Prerequisite of #48931 on lts/1.108.x, hand-ported from the core_helpers move in b51b9b7d43d (#47347), which the line does not carry. tool_fields imports the helper from core_helpers, and the responses bridge now imports it from there too. --- litellm/litellm_core_utils/core_helpers.py | 5 +++++ .../litellm_completion_transformation/transformation.py | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e706bcadef8..eae9f476863 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c import httpx from pydantic import TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason @@ -92,6 +93,10 @@ def normalize_drop_params(value: object) -> bool | None: return None +def drops_unsupported_params(drop_params: object) -> bool: + return litellm.drop_params is True or normalize_drop_params(drop_params) is True + + def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool: normalized: Final = normalize_drop_params(value) if normalized is None and value is not None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 2e7fb6da194..31cd78c7517 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -41,7 +41,7 @@ from litellm._logging import verbose_logger from litellm.caching import InMemoryCache from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -322,10 +322,6 @@ def _dropped_cache_control(_: object) -> None: return None -def drops_unsupported_params(drop_params: object) -> bool: - return litellm.drop_params is True or normalize_drop_params(drop_params) is True - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item From 628f97a9e4336ce8b09de84fe41fc45edb020ebd Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:30:43 -0800 Subject: [PATCH 2/3] fix(mcp)!: answer 400 for a hosted MCP tool a translated route cannot run, drop it under drop_params (#48931) Backport of #48931 to lts/1.108.x. Cherry-picked from 61da6fcb4dc02cd58e39cb8c57ab3c99bea39a4f (main). --- .../prompt_templates/factory.py | 8 +- litellm/litellm_core_utils/tool_fields.py | 151 +++++++++++++++++- litellm/llms/anthropic/chat/transformation.py | 26 ++- .../pass_through/adapters/handler.py | 36 ++++- .../responses_adapters/handler.py | 17 +- .../bedrock/chat/converse_transformation.py | 16 +- .../bedrock/claude_platform/transformation.py | 3 + .../llms/databricks/chat/transformation.py | 26 +-- litellm/llms/deepseek/chat/transformation.py | 23 +-- litellm/llms/gemini/chat/transformation.py | 4 +- litellm/llms/gigachat/chat/transformation.py | 6 +- litellm/llms/ollama/chat/transformation.py | 13 +- .../vertex_and_google_ai_studio_gemini.py | 14 +- .../mcp_security/mcp_security_guardrail.py | 19 ++- litellm/types/llms/anthropic.py | 1 + litellm/utils.py | 31 +++- .../litellm_core_utils/test_tool_fields.py | 96 +++++++++++ .../test_anthropic_chat_transformation.py | 79 +++++++++ .../test_handler_output_config_passthrough.py | 116 +++++++++++++- .../test_responses_adapters_handler.py | 51 ++++++ .../chat/test_converse_transformation.py | 66 +++++++- .../test_databricks_chat_transformation.py | 68 ++++++++ .../chat/test_deepseek_chat_transformation.py | 45 ++++++ .../llms/gigachat/chat/test_transformation.py | 55 +++++++ .../ollama/test_ollama_chat_transformation.py | 67 ++++++++ .../test_ollama_completion_transformation.py | 70 ++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 84 ++++++++++ .../guardrail_hooks/test_mcp_security.py | 151 ++++++++++-------- .../test_litellm_completion_responses.py | 23 +++ .../llm_calls/responses_api.test.tsx | 44 ++++- .../components/llm_calls/responses_api.tsx | 55 ++----- 31 files changed, 1284 insertions(+), 180 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2613e0bbd28..d627edc3fe5 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -29,6 +29,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value from litellm.types.files import get_file_extension_from_mime_type @@ -5547,8 +5548,13 @@ def function_call_prompt( llm_provider: str, drop_params: bool, ) -> list[dict[str, object]]: + prompt_functions: Final = tools_without_hosted_mcp( + functions, model=model, route=llm_provider, drop_params=drop_params + ) + if not prompt_functions: + return messages header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:""" - function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions) + function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions) if not any(_is_system_message(message) for message in messages): return [*messages, {"role": "system", "content": function_prompt}] return [ diff --git a/litellm/litellm_core_utils/tool_fields.py b/litellm/litellm_core_utils/tool_fields.py index f90c8f5d174..5ee96ea9133 100644 --- a/litellm/litellm_core_utils/tool_fields.py +++ b/litellm/litellm_core_utils/tool_fields.py @@ -1,13 +1,16 @@ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence +from copy import copy from dataclasses import dataclass from functools import reduce -from typing import Annotated, Final, TypeAlias +from typing import Annotated, Final, TypeAlias, TypeVar from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params ROUTES_WITH_TOOLS: Final = frozenset( { @@ -162,3 +165,145 @@ def resolve_deployment_tool_fields( if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS: return kwargs return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),)) + + +_ToolT: Final = TypeVar("_ToolT") + +HOSTED_MCP_DROP_WARNING: Final = ( + "Dropping the hosted MCP tool(s) %s for model=%s on %s (drop_params=True) because this route cannot run a " + "hosted MCP tool." +) + + +def _hosted_mcp_server_label(tool: object) -> str | None: + match tool: + case {"type": "mcp", "server_label": str() as server_label}: + return server_label + case {"type": "mcp"}: + return "unlabeled" + case _: + return None + + +def _drop_params_hint(dropped: str) -> str: + return ( + f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ) + + +def _one_log_line(text: str) -> str: + return text.replace("\r", "").replace("\n", "") + + +def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError: + return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400) + + +def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]: + server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None) + if not server_labels: + return tools + named_servers: Final = ", ".join(f"`{label}`" for label in server_labels) + if not drops_unsupported_params(drop_params): + raise _unrunnable_mcp_error( + f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM ' + "run the MCP server for this model, register it on the LiteLLM MCP gateway and send " + '`server_url: "litellm_proxy/mcp/"`. ' + _drop_params_hint("hosted MCP tools"), + model=model, + route=route, + ) + verbose_logger.warning( + HOSTED_MCP_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route) + ) + return [tool for tool in tools if _hosted_mcp_server_label(tool) is None] + + +MCP_SERVERS_DROP_WARNING: Final = ( + "Dropping mcp_servers %s and their mcp_toolset tools for model=%s on %s (drop_params=True) because LiteLLM " + "cannot pass mcp_servers to this route." +) + +ROUTES_WITHOUT_MCP_SERVERS: Final = frozenset( + {"bedrock", "databricks", "deepseek", "gemini", "gigachat", "ollama", "ollama_chat", "vertex_ai", "vertex_ai_beta"} +) + + +def _mcp_server_name(server: object) -> str: + match server: + case {"name": str() as name}: + return name + case _: + return "unnamed" + + +def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]: + if not mcp_servers: + return () + if not isinstance(mcp_servers, list): + return ("unnamed",) + return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers))) + + +def _is_mcp_toolset(tool: object) -> bool: + match tool: + case {"type": "mcp_toolset"}: + return True + case _: + return False + + +def drops_mcp_servers(request_kwargs: Mapping[str, object]) -> bool: + additional_drop_params: Final = request_kwargs.get("additional_drop_params") + return drops_unsupported_params(request_kwargs.get("drop_params")) or ( + isinstance(additional_drop_params, list) and "mcp_servers" in additional_drop_params + ) + + +def tools_without_mcp_servers( + mcp_servers: object, tools: Sequence[_ToolT], model: str, route: str, drop_params: bool +) -> Sequence[_ToolT]: + server_names: Final = _mcp_server_names(mcp_servers) + if not server_names: + return tools + named_servers: Final = ", ".join(f"`{name}`" for name in server_names) + if not drops_unsupported_params(drop_params): + raise _unrunnable_mcp_error( + f"LiteLLM cannot pass `mcp_servers` to {model} on {route}, so the MCP server {named_servers} would not " + 'run. To have LiteLLM run it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` ' + 'tool with `server_url: "litellm_proxy/mcp/"`. ' + _drop_params_hint("`mcp_servers`"), + model=model, + route=route, + ) + verbose_logger.warning( + MCP_SERVERS_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route) + ) + return [tool for tool in tools if not _is_mcp_toolset(tool)] + + +_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"}) +_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object]) + + +def tool_params_without_hosted_mcp( + params: _ParamsT, model: str, route: str, drop_params: bool, runs_hosted_mcp: Callable[[], bool] | None = None +) -> _ParamsT: + tools: Final = params.get("tools") + if not isinstance(tools, list) or not tools: + return params + sent_tools: Final = _TOOLS.validate_python(tools) + if all(_hosted_mcp_server_label(tool) is None for tool in sent_tools) or ( + runs_hosted_mcp is not None and runs_hosted_mcp() + ): + return params + kept_tools: Final = tools_without_hosted_mcp(tools=sent_tools, model=model, route=route, drop_params=drop_params) + if len(kept_tools) == len(sent_tools): + return params + kept_params: Final = copy(params) + if kept_tools: + kept_params["tools"] = list(kept_tools) + return kept_params + dropped_keys: Final = ("tools",) if params.get("functions") else _TOOL_SELECTION_PARAMS + for key in dropped_keys: + _ = kept_params.pop(key, None) + return kept_params diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c20241136b3..6092accc1de 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm +from litellm.anthropic_beta_headers_manager import get_unsupported_headers from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, @@ -35,6 +36,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, as_json_object, @@ -1205,6 +1207,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): *(tool for tool in tools[first_search_tool_index + 1 :] if not _is_openai_tool_search_tool(tool)), ] + def runs_hosted_mcp(self) -> bool: + return ANTHROPIC_BETA_HEADER_VALUES.MCP_CLIENT_2025_04_04.value not in get_unsupported_headers( + self._resolved_provider + ) + def _tools_param_with_openai_tool_search_mapped( self, non_default_params: Mapping[str, object], model: str, drop_params: bool ) -> Mapping[str, object]: @@ -1770,7 +1777,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model: str, drop_params: bool, ) -> dict: - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=non_default_params) + request_params: Final = tool_params_without_hosted_mcp( + non_default_params, + model=model, + route=self._resolved_provider, + drop_params=drop_params, + runs_hosted_mcp=self.runs_hosted_mcp, + ) + is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params) # NB: ``map_openai_params`` deliberately does NOT sanitize tool names # here. Names are the *original* OpenAI names at this stage, and must @@ -1783,7 +1797,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # via ``data = {**optional_params}`` and would 400 with # ``Extra inputs are not permitted``). - for param, value in non_default_params.items(): + for param, value in request_params.items(): if param == "max_tokens" or param == "max_completion_tokens": optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value))) elif param == "tools": @@ -1797,8 +1811,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["mcp_servers"] = mcp_servers elif param == "tool_choice" or param == "parallel_tool_calls": _tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice( - tool_choice=non_default_params.get("tool_choice"), - parallel_tool_use=non_default_params.get("parallel_tool_calls"), + tool_choice=request_params.get("tool_choice"), + parallel_tool_use=request_params.get("parallel_tool_calls"), ) if _tool_choice is not None: @@ -1856,7 +1870,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # `thinking={type: enabled, budget_tokens}` interface a # pre-4.6 model actually supports instead of forwarding a # shape the model will reject. - max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens") + max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens") legacy_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort="medium", model=model, @@ -1949,7 +1963,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ## handle thinking tokens self.update_optional_params_with_thinking_tokens( - non_default_params=non_default_params, optional_params=optional_params + non_default_params=request_params, optional_params=optional_params ) return optional_params diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 7b8c2550838..8c0c065c457 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -14,6 +14,11 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.tool_fields import ( + ROUTES_WITHOUT_MCP_SERVERS, + drops_mcp_servers, + tools_without_mcp_servers, +) from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, ) @@ -588,6 +593,22 @@ class LiteLLMMessagesToCompletionTransformationHandler: Logging as LiteLLMLoggingObject, ) + provider_kwargs: Final = extra_kwargs or {} + custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider") + target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None + strips_mcp_servers: Final = target_provider in ROUTES_WITHOUT_MCP_SERVERS + request_tools: Final = ( + tools_without_mcp_servers( + provider_kwargs.get("mcp_servers"), + tools or (), + model=model, + route=target_provider, + drop_params=drops_mcp_servers(provider_kwargs), + ) + if strips_mcp_servers + else tools + ) + request_data: Final[dict[str, object]] = { "model": model, "messages": messages, @@ -604,10 +625,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - if tool_choice: + if tool_choice and (request_tools or not tools): request_data["tool_choice"] = tool_choice - if tools: - request_data["tools"] = tools + if request_tools: + request_data["tools"] = request_tools if top_k is not None: request_data["top_k"] = top_k if top_p is not None: @@ -625,13 +646,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: if "output_config" in extra_kwargs: request_data["output_config"] = extra_kwargs["output_config"] - custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, - custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + custom_llm_provider=target_provider, ) if openai_request is None: @@ -659,7 +679,11 @@ class LiteLLMMessagesToCompletionTransformationHandler: # Maintainability: when adding a new Anthropic-only request param to # ``AnthropicMessagesRequestOptionalParams``, also extend # ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak. - excluded_keys: Final = ANTHROPIC_ONLY_REQUEST_KEYS | CONVERTED_STREAM_KEYS | {"anthropic_messages"} + excluded_keys: Final = ( + ANTHROPIC_ONLY_REQUEST_KEYS + | CONVERTED_STREAM_KEYS + | ({"anthropic_messages", "mcp_servers"} if strips_mcp_servers else {"anthropic_messages"}) + ) # NOTE: extra_kwargs was already coerced from None to {} at the top of # this method (line ~220). It is guaranteed to be a dict here. for key, value in extra_kwargs.items(): diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 692e5d4c175..3eeaee8ad43 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -16,6 +16,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( ) from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( AllAnthropicMessageValues, @@ -141,6 +142,14 @@ def _build_responses_kwargs( Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + target_provider: Final = forwarded_kwargs.get("custom_llm_provider") + request_tools: Final = tools_without_mcp_servers( + forwarded_kwargs.get("mcp_servers"), + tools or (), + model=model, + route=target_provider if isinstance(target_provider, str) else "this route", + drop_params=drops_mcp_servers(forwarded_kwargs), + ) block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) request_messages: Final[AnthropicRequestMessages] = ( [ @@ -173,10 +182,10 @@ def _build_responses_kwargs( request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - if tool_choice: + if tool_choice and (request_tools or not tools): request_data["tool_choice"] = tool_choice - if tools: - request_data["tools"] = tools + if request_tools: + request_data["tools"] = list(request_tools) if top_p is not None: request_data["top_p"] = top_p if output_format: @@ -213,7 +222,7 @@ def _build_responses_kwargs( responses_kwargs["stream"] = True # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) - excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"} + excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages", "mcp_servers"} for key, value in forwarded_kwargs.items(): if key == "litellm_logging_obj" and value is not None: from litellm.litellm_core_utils.litellm_logging import ( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6984fc1c1cd..943acfda2af 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -45,6 +45,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -1047,20 +1048,23 @@ class AmazonConverseConfig(BaseConfig): model: str, drop_params: bool, ) -> dict: - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) + request_params: Final = tool_params_without_hosted_mcp( + non_default_params, model=model, route="bedrock", drop_params=drop_params + ) + is_thinking_enabled: Final = self.is_thinking_enabled(request_params) base_model: Final = BedrockModelInfo.get_base_model(model) drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model) drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param( model=model, base_model=base_model ) - for param, value in non_default_params.items(): + for param, value in request_params.items(): if param == "response_format" and isinstance(value, dict): optional_params = self._translate_response_format_param( value=value, model=model, optional_params=optional_params, - non_default_params=non_default_params, + non_default_params=request_params, is_thinking_enabled=is_thinking_enabled, ) if param == "max_tokens" or param == "max_completion_tokens": @@ -1113,7 +1117,7 @@ class AmazonConverseConfig(BaseConfig): ), ), model=model, - non_default_params=non_default_params, + non_default_params=request_params, optional_params=optional_params, ) if param == "tool_choice": @@ -1139,7 +1143,7 @@ class AmazonConverseConfig(BaseConfig): and value.get("type") == "adaptive" and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock") ): - max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens") + max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens") legacy_thinking = AnthropicConfig.map_reasoning_effort( reasoning_effort="medium", model=model, @@ -1216,7 +1220,7 @@ class AmazonConverseConfig(BaseConfig): # Nova 2 handles token budgeting differently through reasoningConfig if "gpt-oss" not in model and not self._is_nova_2_model(model): self.update_optional_params_with_thinking_tokens( - non_default_params=non_default_params, optional_params=optional_params + non_default_params=request_params, optional_params=optional_params ) final_is_thinking_enabled: Final = self.is_thinking_enabled(optional_params) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index cea1140bf65..047f9d342e6 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -29,6 +29,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): def should_strip_billing_metadata(self) -> bool: return True + def runs_hosted_mcp(self) -> bool: + return True + def validate_environment( self, headers: dict, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 2145c19a71a..8c96cc4f185 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_internal_message_fields, strip_name_from_message, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict @@ -283,7 +284,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): complete_url: Final = f"{url_base}/chat/completions" return complete_url - def get_supported_openai_params(self, model: str | None = None) -> list: + def get_supported_openai_params(self, model: str | None = None) -> list[str]: return [ "stream", "stop", @@ -389,8 +390,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): drop_params: bool, replace_max_completion_tokens_with_max_tokens: bool = True, ) -> dict: - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) - mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) + request_params: Final = ( + tool_params_without_hosted_mcp(non_default_params, model=model, route="databricks", drop_params=drop_params) + if "claude" in model + else non_default_params + ) + is_thinking_enabled: Final = self.is_thinking_enabled(request_params) + mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params) if "claude" in model: AnthropicConfig.translate_legacy_thinking_for_adaptive_model( model=model, optional_params=mapped_params, custom_llm_provider="databricks", drop_params=drop_params @@ -399,16 +405,16 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): mapped_params["tools"] = self._map_openai_to_dbrx_tool( model=model, tools=mapped_params["tools"], drop_params=drop_params ) - if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens: - mapped_params["max_tokens"] = non_default_params[ + if "max_completion_tokens" in request_params and replace_max_completion_tokens_with_max_tokens: + mapped_params["max_tokens"] = request_params[ "max_completion_tokens" ] # most openai-compatible providers support 'max_tokens' not 'max_completion_tokens' mapped_params.pop("max_completion_tokens", None) - if "response_format" in non_default_params and "claude" in model: + if "response_format" in request_params and "claude" in model: _tool: Final = self.map_response_format_to_databricks_tool( model, - non_default_params["response_format"], + request_params["response_format"], mapped_params, is_thinking_enabled, ) @@ -426,8 +432,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): "response_format", None ) # unsupported for claude models - if json_schema -> convert to tool call - if "reasoning_effort" in non_default_params and self._uses_anthropic_thinking_param(model): - reasoning_effort_value: Final = non_default_params.get("reasoning_effort") + if "reasoning_effort" in request_params and self._uses_anthropic_thinking_param(model): + reasoning_effort_value: Final = request_params.get("reasoning_effort") mapped_thinking: Final = AnthropicConfig._map_reasoning_effort( reasoning_effort=reasoning_effort_value, model=model, @@ -453,7 +459,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): optional_params.pop("reasoning_effort", None) ## handle thinking tokens self.update_optional_params_with_thinking_tokens( - non_default_params=non_default_params, optional_params=mapped_params + non_default_params=request_params, optional_params=mapped_params ) return mapped_params diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index d3319856d29..e1e9694e2ab 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import supports_reasoning, supports_vision @@ -18,17 +19,15 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig class DeepSeekChatConfig(OpenAIGPTConfig): - def get_supported_openai_params(self, model: str) -> list: + def get_supported_openai_params(self, model: str) -> list[str]: """ DeepSeek reasoner models support thinking parameter. """ - params: Final = super().get_supported_openai_params(model) - params.extend(["thinking", "reasoning_effort"]) - return params + return [*super().get_supported_openai_params(model), "thinking", "reasoning_effort"] def map_openai_params( self, - non_default_params: dict, + non_default_params: dict[str, object], optional_params: dict, model: str, drop_params: bool, @@ -44,7 +43,12 @@ class DeepSeekChatConfig(OpenAIGPTConfig): Reference: https://api-docs.deepseek.com/guides/thinking_mode """ # Let parent handle standard params first - optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) + optional_params = super().map_openai_params( + tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params), + optional_params, + model, + drop_params, + ) # Pop thinking/reasoning_effort from optional_params first (parent may have added them) # Then re-add only if valid for DeepSeek @@ -242,6 +246,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): tools: Final = optional_params.get("tools") if not isinstance(tools, list) or not tools: return optional_params + sent_tools: Final[tuple[object, ...]] = tuple(tools) def _is_function_tool(tool: object) -> bool: return isinstance(tool, dict) and tool.get("type") == "function" @@ -266,14 +271,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig): name: Final = function.get("name") return isinstance(name, str) and name in function_tool_names - function_tools: Final = [tool for tool in tools if _is_function_tool(tool)] - if len(function_tools) == len(tools): + function_tools: Final = [tool for tool in sent_tools if _is_function_tool(tool)] + if len(function_tools) == len(sent_tools): return optional_params dropped_types: Final = sorted( { str(tool.get("type")) if isinstance(tool, dict) else type(tool).__name__ - for tool in tools + for tool in sent_tools if not _is_function_tool(tool) } ) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 143652ee77c..79842017c34 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Final, cast +from typing import ClassVar, Final, cast import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -49,6 +49,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): Note: Please make sure to modify the default parameters as required for your use case. """ + _route: ClassVar[str] = "gemini" + temperature: float | None = None max_output_tokens: int | None = None top_p: float | None = None diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 1fe7de0ba85..3ec46cb7e88 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -17,6 +17,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.gigachat.utils import convert_usage, get_api_base from litellm.secret_managers.main import get_secret_str @@ -160,7 +161,10 @@ class GigaChatConfig(BaseConfig): drop_params: bool, ) -> dict: # mutable-ok: base class contract returns dict """Map OpenAI parameters to GigaChat parameters.""" - for param, value in non_default_params.items(): + request_params: Final = tool_params_without_hosted_mcp( + dict(non_default_params), model=model, route="gigachat", drop_params=drop_params + ) + for param, value in request_params.items(): if param == "stream": optional_params["stream"] = value elif param == "temperature": diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 644ab187f15..46b4caad43f 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, cast from httpx._models import Headers, Response -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._uuid import uuid @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.ollama import ( @@ -38,6 +39,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object]) + class OllamaChatConfig(BaseConfig): """ @@ -150,7 +153,13 @@ class OllamaChatConfig(BaseConfig): model: str, drop_params: bool, ) -> dict: - for param, value in non_default_params.items(): + request_params: Final = tool_params_without_hosted_mcp( + _REQUEST_PARAMS.validate_python(non_default_params), + model=model, + route="ollama_chat", + drop_params=drop_params, + ) + for param, value in request_params.items(): if param == "max_tokens" or param == "max_completion_tokens": optional_params["num_predict"] = value if param == "stream": diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 5faf24ac92b..311d23f363d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -8,7 +8,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args import httpx from pydantic import JsonValue @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, ) +from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -279,6 +280,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Note: Please make sure to modify the default parameters as required for your use case. """ + _route: ClassVar[str] = "vertex_ai" + temperature: float | None = None max_output_tokens: int | None = None top_p: float | None = None @@ -1245,9 +1248,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model: str, drop_params: bool, ) -> dict: - self._apply_include_server_side_tool_invocations(non_default_params, optional_params) + request_params: Final = tool_params_without_hosted_mcp( + non_default_params, model=model, route=self._route, drop_params=drop_params + ) + self._apply_include_server_side_tool_invocations(request_params, optional_params) gemini_sampling_params_warned: bool = False - for param, value in non_default_params.items(): + for param, value in request_params.items(): if param == "temperature": if VertexGeminiConfig._is_gemini_3_or_newer(model): if value is not None and value < 1.0: @@ -1317,7 +1323,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value elif param == "parallel_tool_calls": - tools_list = non_default_params.get("tools", non_default_params.get("functions")) + tools_list = request_params.get("tools", request_params.get("functions")) num_tools = len(tools_list) if isinstance(tools_list, list) else 0 # Gemini does not support parallel_tool_calls=False with multiple # tools. Drop the param instead of failing — Responses API clients diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 8482567b6da..9cd5555fe25 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -104,9 +104,20 @@ class MCPSecurityGuardrail(CustomGuardrail): from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy.proxy_server import prisma_client async with global_mcp_server_manager.catalog.operation(): - registry: Final = global_mcp_server_manager.get_registry() - registered_names: Final = set(registry.keys()) - - return frozenset(requested_servers - registered_names) + not_servers: Final = frozenset( + name + for name in requested_servers + if global_mcp_server_manager.get_mcp_server_by_identifier(name) is None + ) + if prisma_client is None: + return not_servers + return frozenset( + [ + name + for name in not_servers + if await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) is None + ] + ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f113f833a46..53abac75162 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -807,6 +807,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): ADVISOR_TOOL_2026_03_01 = "advisor-tool-2026-03-01" PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01" DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03" + MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) diff --git a/litellm/utils.py b/litellm/utils.py index 51dc6db456a..fea6b9d2177 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -93,7 +93,11 @@ from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens -from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields +from litellm.litellm_core_utils.tool_fields import ( + MalformedField, + resolve_deployment_tool_fields, + tools_without_hosted_mcp, +) from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.catalog import decision from litellm.rust_bridge.configuration import Decision @@ -4690,7 +4694,12 @@ def remove_sensitive_keys_from_dict(d: dict) -> dict: return d -def pre_process_optional_params(passed_params: dict, non_default_params: dict, custom_llm_provider: str) -> dict: +_TOOL_LIST: Final = TypeAdapter(list[object]) + + +def pre_process_optional_params( + passed_params: dict, non_default_params: dict, custom_llm_provider: str, model: str, drop_params: bool | None +) -> dict: """For .completion(), preprocess optional params""" optional_params: dict = {} @@ -4747,14 +4756,20 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c and custom_llm_provider not in litellm.openai_compatible_providers ): if custom_llm_provider == "ollama": + if "tools" in non_default_params: + sent_tools: Final = _TOOL_LIST.validate_python(non_default_params.pop("tools")) + non_default_params.pop("tool_choice", None) # causes ollama requests to hang + prompt_tools: Final = tools_without_hosted_mcp( + sent_tools, model=model, route=custom_llm_provider, drop_params=bool(drop_params) + ) + if sent_tools and not prompt_tools: + return optional_params + optional_params["functions_unsupported_model"] = prompt_tools + elif "functions" in non_default_params: + optional_params["functions_unsupported_model"] = non_default_params.pop("functions") # ollama actually supports json output optional_params["format"] = "json" litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt - if "tools" in non_default_params: - optional_params["functions_unsupported_model"] = non_default_params.pop("tools") - non_default_params.pop("tool_choice", None) # causes ollama requests to hang - elif "functions" in non_default_params: - optional_params["functions_unsupported_model"] = non_default_params.pop("functions") elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead optional_params["functions_unsupported_model"] = non_default_params.pop( "tools", non_default_params.pop("functions", None) @@ -4841,6 +4856,8 @@ def get_optional_params( passed_params=passed_params, non_default_params=non_default_params, custom_llm_provider=custom_llm_provider, + model=model, + drop_params=drop_params, ) def _check_valid_arg(supported_params: list[str]): diff --git a/tests/unit/litellm_core_utils/test_tool_fields.py b/tests/unit/litellm_core_utils/test_tool_fields.py index 23d7de98fbc..a99278d18dd 100644 --- a/tests/unit/litellm_core_utils/test_tool_fields.py +++ b/tests/unit/litellm_core_utils/test_tool_fields.py @@ -1,4 +1,5 @@ import datetime +import logging from collections.abc import Callable, Mapping from typing import Final @@ -6,7 +7,13 @@ import pytest import litellm from litellm import Router +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.tool_fields import ( + tool_params_without_hosted_mcp, + tools_without_hosted_mcp, + tools_without_mcp_servers, +) from litellm.utils import async_pre_call_deployment_hook _GOOD_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} @@ -185,3 +192,92 @@ async def test_tools_core_accepts_reach_the_deployment(tools: object) -> None: sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion") assert sent["tools"] is tools + + +def test_a_hosted_mcp_error_names_every_server_and_marks_an_unlabeled_one(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as raised: + tools_without_hosted_mcp( + tools=[ + {"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}, + {"type": "mcp", "server_url": "https://mcp.example.com/mcp"}, + ], + model="gemini-3.8-flash", + route="gemini", + drop_params=False, + ) + assert "the hosted MCP tool `deepwiki`, `unlabeled` " in str(raised.value) + + +_HOSTED_MCP_TOOL: Final = {"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"} + + +def test_dropping_a_hosted_mcp_tool_keeps_the_other_tool_params_and_leaves_the_caller_dict_alone() -> None: + tool_choice: Final = {"type": "function", "function": {"name": "get_weather"}} + sent: Final = { + "tools": [_HOSTED_MCP_TOOL, _GOOD_TOOL], + "tool_choice": tool_choice, + "parallel_tool_calls": False, + "temperature": 0.2, + } + params: Final = tool_params_without_hosted_mcp(sent, model="deepseek-v4-flash", route="deepseek", drop_params=True) + + assert params == { + "tools": [_GOOD_TOOL], + "tool_choice": tool_choice, + "parallel_tool_calls": False, + "temperature": 0.2, + } + assert sent["tools"] == [_HOSTED_MCP_TOOL, _GOOD_TOOL] + + +def test_dropping_every_hosted_mcp_tool_clears_the_tool_selection_params() -> None: + params: Final = tool_params_without_hosted_mcp( + {"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required", "parallel_tool_calls": True, "temperature": 0.2}, + model="deepseek-v4-flash", + route="deepseek", + drop_params=True, + ) + + assert params == {"temperature": 0.2} + + +class _Records(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self.messages: Final[list[str]] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.mark.parametrize( + "drop", + [ + lambda model: tools_without_hosted_mcp( + [{"type": "mcp", "server_label": "wiki\r\nFORGED", "server_url": "https://mcp.example.com/mcp"}], + model=model, + route="deepseek", + drop_params=True, + ), + lambda model: tools_without_mcp_servers( + [{"type": "url", "name": "wiki\r\nFORGED", "url": "https://mcp.example.com/mcp"}], + tools=[], + model=model, + route="bedrock", + drop_params=True, + ), + ], + ids=["hosted_mcp_tool", "mcp_servers"], +) +def test_a_drop_warning_stays_on_one_log_line(drop: Callable[[str], object]) -> None: + records: Final = _Records() + verbose_logger.addHandler(records) + try: + drop("deepseek-v4-flash\nFORGED level=CRITICAL") + finally: + verbose_logger.removeHandler(records) + + assert len(records.messages) == 1 + assert "\n" not in records.messages[0] and "\r" not in records.messages[0] + assert "FORGED level=CRITICAL" in records.messages[0] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 9e70d213f47..8dff6864e8f 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9641,3 +9641,82 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} +_ROUTES_WITHOUT_THE_MCP_CONNECTOR: Final = ( + pytest.param(AmazonAnthropicClaudeConfig, "bedrock", id="bedrock_invoke"), + pytest.param(VertexAIAnthropicConfig, "vertex_ai", id="vertex_ai"), +) + + +@pytest.mark.parametrize("config_class", [AnthropicConfig, AzureAnthropicConfig, BedrockClaudePlatformConfig]) +def test_routes_with_the_mcp_connector_send_a_hosted_mcp_tool_as_an_mcp_server(monkeypatch, config_class): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=False, + ) + assert [server["url"] for server in mapped["mcp_servers"]] == [_HOSTED_MCP_TOOL["server_url"]] + assert [tool["name"] for tool in mapped["tools"]] == ["get_weather"] + + +@pytest.mark.parametrize(("config_class", "route"), _ROUTES_WITHOUT_THE_MCP_CONNECTOR) +def test_a_hosted_mcp_tool_answers_400_on_claude_routes_without_the_mcp_connector(monkeypatch, config_class, route): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + config_class().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert f"claude-haiku-5-5 on {route} " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +@pytest.mark.parametrize(("config_class", "route"), _ROUTES_WITHOUT_THE_MCP_CONNECTOR) +def test_claude_routes_without_the_mcp_connector_drop_a_hosted_mcp_tool_under_drop_params( + monkeypatch, caplog, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=True, + ) + assert [tool["name"] for tool in mapped["tools"]] == ["get_weather"] + assert "mcp_servers" not in mapped + assert any( + "`deepwiki`" in record.getMessage() and f" on {route} (" in record.getMessage() for record in caplog.records + ) + + +@pytest.mark.parametrize(("config_class", "route"), _ROUTES_WITHOUT_THE_MCP_CONNECTOR) +def test_claude_routes_without_the_mcp_connector_send_no_tools_when_drop_params_drops_every_one( + monkeypatch, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required"}, + optional_params={}, + model="claude-haiku-5-5", + drop_params=True, + ) + assert "tools" not in mapped + assert "mcp_servers" not in mapped + assert "tool_choice" not in mapped diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py index 03553e65ad4..55287fa070d 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py @@ -27,11 +27,14 @@ Tests cover (consolidating PRs #23706 and #22727): import os import sys +from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch import pytest +import litellm + # Anchor sys.path to this file's location — not the working-directory-relative # pattern Greptile flagged on PR #23706. Resolves correctly regardless of # where pytest is invoked from. @@ -67,8 +70,8 @@ def _call_prepare(extra_kwargs, model="gpt-4o", output_format=None, **overrides) system=None, temperature=None, thinking=None, - tool_choice=None, - tools=None, + tool_choice=overrides.get("tool_choice"), + tools=overrides.get("tools"), top_k=None, top_p=None, output_format=output_format, @@ -263,3 +266,112 @@ async def test_messages_layer_converted_stream_flag_still_gets_a_message_back(co ) assert response["content"][0]["text"] == "hi" + + +_DEEPWIKI_SERVER: Final = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"} + + +@pytest.mark.parametrize( + ("mcp_servers", "named"), + [ + ([_DEEPWIKI_SERVER], "`deepwiki`"), + ([_DEEPWIKI_SERVER, {"type": "url", "url": "https://mcp.example.com/mcp"}], "`deepwiki`, `unnamed`"), + ({"name": "deepwiki"}, "`unnamed`"), + ], + ids=["named", "one-unnamed", "not-a-list"], +) +def test_mcp_servers_a_chat_route_cannot_run_answer_400_naming_each_server( + monkeypatch: pytest.MonkeyPatch, mcp_servers: object, named: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as raised: + _call_prepare( + extra_kwargs={"mcp_servers": mcp_servers, "custom_llm_provider": "gemini"}, model="gemini-3.8-flash" + ) + + assert raised.value.status_code == 400 + assert f"to gemini-3.8-flash on gemini, so the MCP server {named} would not run" in str(raised.value) + + +@pytest.mark.parametrize( + ("route", "forwards"), + [("litellm_proxy", True), ("hosted_vllm", True), ("bedrock", False), ("vertex_ai", False)], +) +def test_mcp_servers_reach_routes_that_carry_them_and_answer_400_elsewhere( + monkeypatch: pytest.MonkeyPatch, route: str, forwards: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + extra_kwargs: Final = {"mcp_servers": [_DEEPWIKI_SERVER], "custom_llm_provider": route} + if not forwards: + with pytest.raises(litellm.UnsupportedParamsError, match=f"on {route}, so the MCP server `deepwiki`"): + _call_prepare(extra_kwargs=extra_kwargs, model="some-model") + return + + completion_kwargs, _ = _call_prepare(extra_kwargs=extra_kwargs, model="some-model") + + assert completion_kwargs["mcp_servers"] == [_DEEPWIKI_SERVER] + + +_DEEPWIKI_TOOLSET: Final = {"type": "mcp_toolset", "mcp_server_name": "deepwiki"} +_WEATHER_TOOL: Final = { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, +} + + +@pytest.mark.parametrize( + ("tools", "kept_tool_names"), + [([_DEEPWIKI_TOOLSET, _WEATHER_TOOL], ["get_weather"]), ([_DEEPWIKI_TOOLSET], None)], + ids=["with-function", "toolset-only"], +) +def test_drop_params_drops_the_mcp_toolset_tools_of_dropped_mcp_servers( + monkeypatch: pytest.MonkeyPatch, tools: list[dict[str, object]], kept_tool_names: list[str] | None +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + completion_kwargs, _ = _call_prepare( + extra_kwargs={"mcp_servers": [_DEEPWIKI_SERVER], "custom_llm_provider": "gemini"}, + model="gemini-3.8-flash", + tools=tools, + tool_choice={"type": "auto"}, + ) + + sent_tools: Final = completion_kwargs.get("tools") + sent_names: Final = [tool["function"]["name"] for tool in sent_tools] if sent_tools else None + assert sent_names == kept_tool_names + assert ("tool_choice" in completion_kwargs) == bool(kept_tool_names) + + +_DROP_FORMS: Final[Mapping[str, Mapping[str, object]]] = { + "request": {"drop_params": True}, + "global": {}, + "additional_drop_params": {"additional_drop_params": ["mcp_servers"]}, +} + + +@pytest.mark.parametrize("drop_params", list(_DROP_FORMS)) +def test_drop_params_drops_mcp_servers_instead_of_forwarding_them( + monkeypatch: pytest.MonkeyPatch, drop_params: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params == "global") + completion_kwargs, _ = _call_prepare( + extra_kwargs={ + "mcp_servers": [_DEEPWIKI_SERVER], + "custom_llm_provider": "gemini", + **_DROP_FORMS[drop_params], + }, + model="gemini-3.8-flash", + ) + + assert "mcp_servers" not in completion_kwargs + assert completion_kwargs["messages"] == MESSAGES + + +@pytest.mark.parametrize("mcp_servers", [None, []], ids=["absent", "empty"]) +def test_no_mcp_servers_needs_no_drop_params(monkeypatch: pytest.MonkeyPatch, mcp_servers: object) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + completion_kwargs, _ = _call_prepare( + extra_kwargs={"mcp_servers": mcp_servers, "custom_llm_provider": "gemini"}, model="gemini-3.8-flash" + ) + + assert "mcp_servers" not in completion_kwargs diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 453429df084..6aec78bdb97 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -633,3 +633,54 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +_DEEPWIKI_SERVER: Final = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"} + + +def test_mcp_servers_the_responses_bridge_cannot_run_answer_400(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as rejected: + _build_responses_kwargs( + model="gpt-5.6", + extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER]}, + max_tokens=1024, + messages=MESSAGES, + ) + + assert rejected.value.status_code == 400 + assert "to gpt-5.6 on openai, so the MCP server `deepwiki` would not run" in rejected.value.message + + +@pytest.mark.parametrize( + "drop_form", + [{"drop_params": True}, {"additional_drop_params": ["mcp_servers"]}], + ids=["drop_params", "additional_drop_params"], +) +def test_drop_params_keeps_mcp_servers_off_the_responses_request( + monkeypatch: pytest.MonkeyPatch, drop_form: Mapping[str, object] +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + responses_kwargs: Final = _build_responses_kwargs( + model="gpt-5.6", + extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER], **drop_form}, + max_tokens=1024, + messages=MESSAGES, + ) + + assert "mcp_servers" not in responses_kwargs + assert responses_kwargs["input"] + + +def test_drop_params_drops_the_mcp_toolset_tools_of_dropped_mcp_servers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + weather_tool: Final = {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}} + responses_kwargs: Final = _build_responses_kwargs( + model="gpt-5.6", + extra_kwargs={**OPENAI_API, "mcp_servers": [_DEEPWIKI_SERVER], "drop_params": True}, + max_tokens=1024, + messages=MESSAGES, + tools=[{"type": "mcp_toolset", "mcp_server_name": "deepwiki"}, weather_tool], + ) + + assert [tool["name"] for tool in responses_kwargs["tools"]] == ["get_weather"] diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6c996a7ce75..8c1815a779a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -3,7 +3,7 @@ import copy import json import os import re -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -10147,3 +10147,67 @@ def test_map_openai_params_drops_the_openai_tool_search_tool_and_every_deferral_ ) assert [tool["function"]["name"] for tool in mapped["tools"]] == ["get_weather", "get_stock_price"] assert all("defer_loading" not in tool and "defer_loading" not in tool["function"] for tool in mapped["tools"]) + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def _converse_request_for_tools( + model: str, non_default_params: Mapping[str, object], drop_params: bool +) -> Mapping[str, object]: + config: Final = AmazonConverseConfig() + return config._transform_request_helper( + model=model, + system_content_blocks=[], + optional_params=config.map_openai_params( + non_default_params=dict(non_default_params), optional_params={}, model=model, drop_params=drop_params + ), + messages=[{"role": "user", "content": "hi"}], + ) + + +@pytest.mark.parametrize( + "model", ["us.openai.gpt-oss-120b-1:0", "us.anthropic.claude-haiku-5-5", "us.amazon.nova-2-lite-v1:0"] +) +def test_a_hosted_mcp_tool_answers_400_on_converse_without_drop_params(monkeypatch, model): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + _converse_request_for_tools(model, {"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, drop_params=False) + assert raised.value.status_code == 400 + assert model in str(raised.value) + assert "`deepwiki`" in str(raised.value) + assert 'server_url: "litellm_proxy/mcp/"' in str(raised.value) + assert "drop_params: true" in str(raised.value) + + +def test_converse_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + model: Final = "us.openai.gpt-oss-120b-1:0" + with caplog.at_level("WARNING"): + request: Final = _converse_request_for_tools( + model, {"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, drop_params=True + ) + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["get_weather"] + assert any( + "`deepwiki`" in record.getMessage() and f"model={model} on bedrock (" in record.getMessage() + for record in caplog.records + ) + + +def test_converse_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _converse_request_for_tools( + "us.openai.gpt-oss-120b-1:0", {"tools": [_HOSTED_MCP_TOOL], "tool_choice": "auto"}, drop_params=True + ) + assert "toolConfig" not in request + assert "toolChoice" not in json.dumps(request) + assert "tool_choice" not in json.dumps(request) diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 650517c92ca..d56bdeefda6 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2447,3 +2447,71 @@ def test_databricks_claude_drops_the_openai_tool_search_tool_under_drop_params(m assert mapped["tools"] == [ {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}} ] + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def test_databricks_claude_answers_400_for_a_hosted_mcp_tool_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + DatabricksConfig().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert "databricks-claude-opus-5-5 on databricks " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +def test_databricks_claude_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert [tool["function"]["name"] for tool in mapped["tools"]] == ["get_weather"] + assert any( + "`deepwiki`" in record.getMessage() and " on databricks (" in record.getMessage() for record in caplog.records + ) + + +def test_databricks_claude_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required"}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert "tools" not in mapped + assert "tool_choice" not in mapped + + +def test_databricks_claude_still_sends_max_completion_tokens_as_max_tokens_when_dropping_a_hosted_mcp_tool( + monkeypatch, +): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL], "max_completion_tokens": 64}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert mapped["max_tokens"] == 64 + assert "max_completion_tokens" not in mapped + assert [tool["function"]["name"] for tool in mapped["tools"]] == ["get_weather"] diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index dd99ad8bf91..ab38f4fc41f 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -907,3 +907,48 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "https://gateway.example.com/deepseek/chat/completions" ) + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} + + +def test_a_hosted_mcp_tool_answers_400_on_deepseek_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + DeepSeekChatConfig().map_openai_params( + {"tools": [_function_tool("shell"), _HOSTED_MCP_TOOL]}, {}, "deepseek-v4-flash", False + ) + assert raised.value.status_code == 400 + assert "deepseek-v4-flash on deepseek " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +def test_deepseek_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + result: Final = DeepSeekChatConfig().map_openai_params( + {"tools": [_HOSTED_MCP_TOOL, _function_tool("shell")], "tool_choice": "auto"}, {}, "deepseek-v4-flash", True + ) + assert result["tools"] == [_function_tool("shell")] + assert result["tool_choice"] == "auto" + assert any( + "`deepwiki`" in record.getMessage() and " on deepseek (" in record.getMessage() for record in caplog.records + ) + + +def test_deepseek_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = DeepSeekChatConfig().map_openai_params( + {"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required", "parallel_tool_calls": True}, + {}, + "deepseek-v4-flash", + True, + ) + assert "tools" not in result + assert "tool_choice" not in result + assert "parallel_tool_calls" not in result diff --git a/tests/unit/llms/gigachat/chat/test_transformation.py b/tests/unit/llms/gigachat/chat/test_transformation.py index 9116a4d1979..4d8a916ea87 100644 --- a/tests/unit/llms/gigachat/chat/test_transformation.py +++ b/tests/unit/llms/gigachat/chat/test_transformation.py @@ -7,10 +7,12 @@ Run with: pytest tests/unit/llms/gigachat/chat/test_transformation.py -v import asyncio import importlib +from typing import Final import pytest import litellm +from litellm.llms.gigachat.chat.transformation import GigaChatConfig from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -575,3 +577,56 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), "cohere_key": getattr(litellm, "cohere_key", None), } + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def test_a_hosted_mcp_tool_answers_400_on_gigachat_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + GigaChatConfig().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="GigaChat-2-Max", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert "GigaChat-2-Max on gigachat " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + + +def test_gigachat_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool(monkeypatch, caplog): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = GigaChatConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="GigaChat-2-Max", + drop_params=True, + ) + assert [function["name"] for function in mapped["functions"]] == ["get_weather"] + assert any( + "`deepwiki`" in record.getMessage() and " on gigachat (" in record.getMessage() for record in caplog.records + ) + + +def test_gigachat_sends_no_functions_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = GigaChatConfig().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "auto"}, + optional_params={}, + model="GigaChat-2-Max", + drop_params=True, + ) + assert "functions" not in mapped + assert "function_call" not in mapped diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 1a0bbe171a9..ae75c40d32a 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -5,6 +5,7 @@ import sys from typing import Final, cast import pytest +import respx from pydantic import BaseModel sys.path.insert( @@ -1108,3 +1109,69 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/chat" ) + + +_DEEPWIKI_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, +} + + +def _ollama_chat_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("http://ollama.test:11434/api/chat").respond( + json={ + "model": "qwen3:0.6b", + "message": {"role": "assistant", "content": "Sunny"}, + "done": True, + "prompt_eval_count": 5, + "eval_count": 3, + } + ) + + +def test_ollama_chat_answers_400_for_a_hosted_mcp_tool_it_cannot_run( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = _ollama_chat_route(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.completion( + model="ollama_chat/qwen3:0.6b", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=[_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], + api_base="http://ollama.test:11434", + ) + + assert raised.value.status_code == 400 + assert "qwen3:0.6b on ollama_chat cannot run the hosted MCP tool `deepwiki`" in str(raised.value) + assert not route.called + + +@pytest.mark.parametrize( + ("tools", "sent_tools"), + [([_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], [_WEATHER_TOOL]), ([_DEEPWIKI_MCP_TOOL], None)], + ids=["beside-a-function", "alone"], +) +def test_ollama_chat_drop_params_sends_only_the_tools_ollama_can_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[object], sent_tools: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = _ollama_chat_route(respx_mock) + + response: Final = litellm.completion( + model="ollama_chat/qwen3:0.6b", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=tools, + api_base="http://ollama.test:11434", + drop_params=True, + ) + + assert json.loads(route.calls.last.request.content).get("tools") == sent_tools + assert response.choices[0].message.content == "Sunny" diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 248e745ebff..0cf18ff6e0d 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -990,3 +990,73 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/generate" ) + + +_DEEPWIKI_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} + + +def _ollama_generate_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("http://ollama.test:11434/api/generate").respond( + json={"model": "llama3.2", "response": "Sunny", "done": True, "prompt_eval_count": 5, "eval_count": 3} + ) + + +def test_ollama_answers_400_for_a_hosted_mcp_tool_instead_of_pasting_it_into_the_prompt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "add_function_to_prompt", False) + route: Final = _ollama_generate_route(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.completion( + model="ollama/llama3.2", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=[_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], + api_base="http://ollama.test:11434", + ) + + assert raised.value.status_code == 400 + assert "cannot run the hosted MCP tool `deepwiki`" in str(raised.value) + assert not route.called + + +@pytest.mark.parametrize( + ("tools", "prompt_names_weather"), + [([_DEEPWIKI_MCP_TOOL, _WEATHER_TOOL], True), ([_DEEPWIKI_MCP_TOOL], False)], + ids=["beside-a-function", "alone"], +) +def test_ollama_drop_params_keeps_a_hosted_mcp_tool_out_of_the_prompt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[object], prompt_names_weather: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "add_function_to_prompt", False) + route: Final = _ollama_generate_route(respx_mock) + + litellm.completion( + model="ollama/llama3.2", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=tools, + api_base="http://ollama.test:11434", + drop_params=True, + ) + + body: Final = json.loads(route.calls.last.request.content) + prompt: Final = body["prompt"] + assert (body.get("format") == "json") is prompt_names_weather + assert "deepwiki" not in prompt + assert ("get_weather" in prompt) is prompt_names_weather + assert prompt.startswith("### User:\nweather in Paris?") + assert prompt_names_weather or prompt == "### User:\nweather in Paris?\n\n" diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3d7296ffa7b..e014b620dec 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -9887,3 +9887,87 @@ async def test_gemini_sends_a_boolean_for_a_gateway_default_written_as_text_or_n sent: Final = await _sent_thinking_config("gemini-3.8-flash", {"reasoning_effort": "high"}) assert sent["includeThoughts"] is expected + + +_HOSTED_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", +} +_WEATHER_FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} +_GEMINI_ROUTES: Final = ( + pytest.param(VertexGeminiConfig, "vertex_ai", id="vertex_ai"), + pytest.param(GoogleAIStudioGeminiConfig, "gemini", id="gemini"), +) + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_a_hosted_mcp_tool_answers_400_on_gemini_without_drop_params(monkeypatch, config_class, route): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: + config_class().map_openai_params( + non_default_params={"tools": [_WEATHER_FUNCTION_TOOL, _HOSTED_MCP_TOOL]}, + optional_params={}, + model="gemini-3.8-flash", + drop_params=False, + ) + assert raised.value.status_code == 400 + assert f"gemini-3.8-flash on {route} " in str(raised.value) + assert "`deepwiki`" in str(raised.value) + assert "litellm_proxy/mcp/" in str(raised.value) + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_gemini_drops_a_hosted_mcp_tool_under_drop_params_and_keeps_the_function_tool( + monkeypatch, caplog, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + with caplog.at_level("WARNING"): + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL, _WEATHER_FUNCTION_TOOL]}, + optional_params={}, + model="gemini-3.8-flash", + drop_params=True, + ) + assert "deepwiki" not in json.dumps(mapped) + assert "get_weather" in json.dumps(mapped["tools"]) + assert any( + "`deepwiki`" in record.getMessage() and f" on {route} (" in record.getMessage() for record in caplog.records + ) + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_gemini_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(monkeypatch, config_class, route): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={"tools": [_HOSTED_MCP_TOOL], "tool_choice": "required"}, + optional_params={}, + model="gemini-3.8-flash", + drop_params=True, + ) + assert "tools" not in mapped + assert "tool_choice" not in mapped + + +@pytest.mark.parametrize(("config_class", "route"), _GEMINI_ROUTES) +def test_gemini_keeps_tool_choice_none_for_legacy_functions_after_dropping_every_hosted_mcp_tool( + monkeypatch, config_class, route +): + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = config_class().map_openai_params( + non_default_params={ + "tools": [_HOSTED_MCP_TOOL], + "functions": [_WEATHER_FUNCTION_TOOL["function"]], + "tool_choice": "none", + }, + optional_params={}, + model="gemini-3.8-flash", + drop_params=True, + ) + assert "deepwiki" not in json.dumps(mapped) + assert "get_weather" in json.dumps(mapped["tools"]) + assert mapped["tool_choice"] == {"functionCallingConfig": {"mode": "NONE"}} diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py index e4c7bcba951..6501526634c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py @@ -6,19 +6,24 @@ and allows requests with only registered servers. Covers both /chat/completions and /responses API paths (same pre_call_hook logic, different call_type). """ -from typing import Literal -from unittest.mock import MagicMock, patch +from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException import litellm +from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.mcp_security import initialize_guardrail from litellm.proxy.guardrails.guardrail_hooks.mcp_security.mcp_security_guardrail import ( MCPSecurityGuardrail, ) +from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.guardrails import Guardrail, GuardrailEventHooks, LitellmParams +from litellm.types.mcp import MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.types.mcp_server.mcp_toolset import MCPToolset @pytest.fixture @@ -60,93 +65,73 @@ class TestExtractMCPServerNames: assert names == set() +def _manager_with_saved_server() -> MCPServerManager: + manager: Final = MCPServerManager() + server: Final = MCPServer( + server_id="3f6b2c1e-saved-zapier", + name="zapier", + server_name="zapier", + alias="zap", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + ) + manager.registry[server.server_id] = server + return manager + + class TestMCPSecurityGuardrailPreCall: @pytest.mark.asyncio - @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) - async def test_blocks_unregistered_server_chat_completions( - self, mock_manager, guardrail - ): - """Simulates /chat/completions with an unregistered MCP server.""" - mock_manager.get_registry.return_value = {"zapier": MagicMock()} - + @pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) + async def test_blocks_unregistered_server(self, guardrail, call_type): data = { "tools": [ {"type": "mcp", "server_url": "litellm_proxy/mcp/zapier"}, {"type": "mcp", "server_url": "litellm_proxy/mcp/evil_server"}, ], "model": "gpt-4", - "messages": [{"role": "user", "content": "hi"}], "guardrails": ["test-mcp-security"], } - with pytest.raises(HTTPException) as exc_info: + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + _manager_with_saved_server(), + ), + pytest.raises(HTTPException) as exc_info, + ): await guardrail.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), cache=MagicMock(), data=data, - call_type="acompletion", + call_type=call_type, ) assert exc_info.value.status_code == 400 - assert "evil_server" in str(exc_info.value.detail) + assert exc_info.value.detail["unregistered_servers"] == ["evil_server"] @pytest.mark.asyncio - @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) - async def test_blocks_unregistered_server_responses_api( - self, mock_manager, guardrail - ): - """Simulates /responses with an unregistered MCP server.""" - mock_manager.get_registry.return_value = {"github": MagicMock()} - + @pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) + @pytest.mark.parametrize("reference", ["zapier", "zap", "3f6b2c1e-saved-zapier"]) + async def test_allows_saved_server_by_any_name_the_gateway_routes(self, guardrail, call_type, reference): data = { - "tools": [ - {"type": "mcp", "server_url": "litellm_proxy/mcp/unknown_server"}, - ], - "model": "gpt-4o", - "input": "What can you do?", - "guardrails": ["test-mcp-security"], - } - - with pytest.raises(HTTPException) as exc_info: - await guardrail.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth(), - cache=MagicMock(), - data=data, - call_type="aresponses", - ) - assert exc_info.value.status_code == 400 - assert "unknown_server" in str(exc_info.value.detail) - - @pytest.mark.asyncio - @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) - async def test_allows_registered_servers(self, mock_manager, guardrail): - """All MCP servers are registered - request passes through.""" - mock_manager.get_registry.return_value = { - "zapier": MagicMock(), - "github": MagicMock(), - } - - data = { - "tools": [ - {"type": "mcp", "server_url": "litellm_proxy/mcp/zapier"}, - {"type": "mcp", "server_url": "litellm_proxy/mcp/github"}, - ], + "tools": [{"type": "mcp", "server_url": f"litellm_proxy/mcp/{reference}"}], "model": "gpt-4", - "messages": [{"role": "user", "content": "hi"}], "guardrails": ["test-mcp-security"], } - result = await guardrail.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth(), - cache=MagicMock(), - data=data, - call_type="acompletion", - ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + _manager_with_saved_server(), + ), + ): + result = await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=MagicMock(), + data=data, + call_type=call_type, + ) assert result == data @pytest.mark.asyncio @@ -221,11 +206,13 @@ async def test_guardrail_observes_saved_server_creation_and_deletion_on_another_ prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(side_effect=([row], [])) prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcptoolsettable.find_first = AsyncMock(return_value=None) data = {"tools": [{"type": "mcp", "server_url": "litellm_proxy/mcp/peer-server"}], "guardrails": ["test-mcp-security"]} with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), ): result = await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) @@ -234,3 +221,37 @@ async def test_guardrail_observes_saved_server_creation_and_deletion_on_another_ await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) assert exc.value.status_code == 400 assert exc.value.detail["unregistered_servers"] == ["peer-server"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) +async def test_allows_saved_toolset_by_name_and_blocks_unknown_name(guardrail, call_type): + saved_toolset: Final = MCPToolset(toolset_id="7c1d-research", toolset_name="research") + + async def find_toolset(*, where: dict[str, str]) -> MCPToolset | None: + return saved_toolset if where["toolset_name"] == saved_toolset.toolset_name else None + + prisma: Final = MagicMock() + prisma.writer_db = prisma.db + prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcptoolsettable.find_first = AsyncMock(side_effect=find_toolset) + data: Final = { + "tools": [ + {"type": "mcp", "server_url": "litellm_proxy/mcp/research"}, + {"type": "mcp", "server_url": "litellm_proxy/mcp/evil_server"}, + ], + "guardrails": ["test-mcp-security"], + } + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + MCPServerManager(), + ), + pytest.raises(HTTPException) as exc, + ): + await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) + assert exc.value.detail["unregistered_servers"] == ["evil_server"] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 79d5ec5d877..4fb2a4af571 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6536,3 +6536,26 @@ async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chun assert incomplete_event.response.usage is not None assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text) assert incomplete_event.response.usage.input_tokens > 0 + + +def test_a_hosted_mcp_tool_sent_to_responses_on_bedrock_converse_answers_400(monkeypatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.responses( + model="bedrock/converse/us.openai.gpt-oss-120b-1:0", + input="What does the deepwiki MCP server say about litellm?", + tools=[ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", + } + ], + aws_access_key_id="AKIAHOSTEDMCP", + aws_secret_access_key="hosted-mcp-secret", + aws_region_name="us-east-1", + ) + assert raised.value.status_code == 400 + assert "`deepwiki`" in str(raised.value) + assert "litellm_proxy/mcp/" in str(raised.value) diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 2d98b29173d..ef630dd6946 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -138,6 +138,46 @@ describe("responses_api", () => { expect(onTimingData).not.toHaveBeenCalled(); }); + it.each([ + [["__all__"], [], [{ server_label: "litellm", server_url: "litellm_proxy/mcp" }]], + [ + ["server-1"], + [{ server_id: "server-1", server_name: "my server" }], + [{ server_label: "my server", server_url: "litellm_proxy/mcp/my server" }], + ], + ])( + "sends MCP selection %j as gateway references the proxy runs for any model", + async (selected, servers, expected) => { + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gemini-pro", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + selected, + undefined, // previousResponseId + undefined, // onResponseId + undefined, // onMCPEvent + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + servers as never, + ); + + expect(mockResponsesCreate.mock.calls[0][0].tools).toEqual( + expected.map((tool) => ({ type: "mcp", require_approval: "never", ...tool })), + ); + }, + ); + it("should report total latency in both streaming and non-streaming modes", async () => { const onTotalLatency = vi.fn(); const callWithStreaming = (streamingEnabled: boolean) => @@ -406,14 +446,14 @@ describe("responses_api", () => { { type: "mcp", server_label: "Alpha", - server_url: "https://backend.example.com/mcp/Alpha", + server_url: "litellm_proxy/mcp/Alpha", require_approval: "never", allowed_tools: ["toolA"], }, { type: "mcp", server_label: "Beta", - server_url: "https://backend.example.com/mcp/Beta", + server_url: "litellm_proxy/mcp/Beta", require_approval: "never", allowed_tools: ["toolB", "toolC"], }, diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 1f717e488e9..82fcb699fb1 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -1,11 +1,12 @@ import openai from "openai"; import { MessageType } from "../chat_ui/types"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; -import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking"; +import { getGatewayBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; import { parseUsageCost } from "./usage_cost"; import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers"; +import { buildMcpToolBlocks, type BuildMcpToolBlocksArgs } from "./mcp_tool_blocks"; import type { MCPEvent } from "@/components/mcp_tools/types"; import { MCPServer, MCPToolset } from "@/components/mcp_tools/types"; import { @@ -103,7 +104,6 @@ export async function makeOpenAIResponsesRequest( } const proxyBaseUrl = customBaseUrl || getGatewayBaseUrl(); - const mcpBaseUrl = customBaseUrl || getProxyBaseUrl(); const headers = buildPlaygroundHeaders(tags, customHeaders); const client = new openai.OpenAI({ @@ -137,50 +137,13 @@ export async function makeOpenAIResponsesRequest( }); // Build tools array - const tools: any[] = []; - - // Add MCP servers if selected - if (selectedMCPServers && selectedMCPServers.length > 0) { - if (selectedMCPServers.includes("__all__")) { - // All MCP Servers selected - tools.push({ - type: "mcp", - server_label: "litellm", - server_url: `${mcpBaseUrl}/mcp`, - require_approval: "never", - }); - } else { - // Individual servers/toolsets selected - create one entry per item - selectedMCPServers.forEach((serverId) => { - if (serverId.startsWith("toolset:")) { - // Toolset: same /{name}/mcp pattern as individual servers - const toolsetId = serverId.slice("toolset:".length); - const toolset = mcpToolsets?.find((t) => t.toolset_id === toolsetId); - const toolsetName = toolset?.toolset_name || toolsetId; - tools.push({ - type: "mcp", - server_label: toolsetName, - server_url: `${mcpBaseUrl}/mcp/${encodeURIComponent(toolsetName)}`, - require_approval: "never", - }); - } else { - const server = mcpServers?.find((s) => s.server_id === serverId); - // Use server_name for both routing and labelling. server_name is the - // unique registered identifier; aliases can collide across servers. - const routeName = server?.server_name || serverId; - const allowedTools = mcpServerToolRestrictions?.[serverId] || []; - - tools.push({ - type: "mcp", - server_label: routeName, // unique per request — collisions cause silent tool-routing failures - server_url: `${mcpBaseUrl}/mcp/${encodeURIComponent(routeName)}`, - require_approval: "never", - ...(allowedTools.length > 0 ? { allowed_tools: allowedTools } : {}), - }); - } - }); - } - } + const mcpSelection: BuildMcpToolBlocksArgs = { + selectedMCPServers, + mcpServers, + mcpToolsets, + mcpServerToolRestrictions, + }; + const tools: any[] = buildMcpToolBlocks(mcpSelection); // Add code_interpreter tool if enabled (OpenAI auto-creates container) if (codeInterpreterEnabled) { From 22563a80d1cb41d954795365c596b7a1fa9e9831 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:50:30 -0800 Subject: [PATCH 3/3] fix(mcp): keep DeepSeek tool cleanup and the guardrail policy result on lts/1.108.x DeepSeek now sends the params through its own tool cleanup when a hosted MCP tool is dropped, so a tool_choice naming that tool and the tool selection params beside legacy functions are cleared as the line does today. The MCP security guardrail uses the gateway's saved toolset check, so a failed read reports the name as unregistered with the policy result instead of raising. Also fixes three lint errors the line already has, two import orders and one duplicate test import, since make check cannot pass on this line without them. --- litellm/llms/deepseek/chat/transformation.py | 7 ++++- .../redis_update_buffer.py | 1 - .../mcp_security/mcp_security_guardrail.py | 12 ++------ .../mcp/litellm_proxy_mcp_handler.py | 4 +-- litellm/utils.py | 2 +- .../test_streaming_chunk_builder_utils.py | 1 - .../chat/test_deepseek_chat_transformation.py | 30 +++++++++++++++++++ .../guardrail_hooks/test_mcp_security.py | 27 +++++++++++++++++ 8 files changed, 68 insertions(+), 16 deletions(-) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index e1e9694e2ab..a742cc6c4de 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -42,9 +42,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig): Reference: https://api-docs.deepseek.com/guides/thinking_mode """ + params_without_hosted_mcp: Final = tool_params_without_hosted_mcp( + non_default_params, model=model, route="deepseek", drop_params=drop_params + ) # Let parent handle standard params first optional_params = super().map_openai_params( - tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params), + params_without_hosted_mcp + if params_without_hosted_mcp is non_default_params + else self._drop_unsupported_tools(non_default_params), optional_params, model, drop_params, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 22f55ea9153..8fbaa7e3a9a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -13,7 +13,6 @@ from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter - from redis.exceptions import RedisError from litellm._internal_context import with_service_target diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 9cd5555fe25..66f493bcea3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -18,6 +18,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.proxy._types import UserAPIKeyAuth from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LITELLM_PROXY_MCP_SERVER_URL_PREFIX, + saved_toolset_exists, ) from litellm.types.guardrails import GuardrailEventHooks @@ -104,7 +105,6 @@ class MCPSecurityGuardrail(CustomGuardrail): from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy.proxy_server import prisma_client async with global_mcp_server_manager.catalog.operation(): not_servers: Final = frozenset( @@ -112,12 +112,4 @@ class MCPSecurityGuardrail(CustomGuardrail): for name in requested_servers if global_mcp_server_manager.get_mcp_server_by_identifier(name) is None ) - if prisma_client is None: - return not_servers - return frozenset( - [ - name - for name in not_servers - if await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) is None - ] - ) + return frozenset([name for name in not_servers if not await saved_toolset_exists(name)]) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index b711d051055..1dfa4b2abcb 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -105,7 +105,7 @@ def _registry_serves(name: str, servers: Collection[MCPServer]) -> bool: ) -async def _toolset_exists(name: str) -> bool: +async def saved_toolset_exists(name: str) -> bool: try: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, @@ -124,7 +124,7 @@ async def _toolset_exists(name: str) -> bool: async def _gateway_served_names( names: Collection[str], servers: Callable[[], Collection[MCPServer]] = _registered_mcp_servers, - toolset_exists: Callable[[str], Awaitable[bool]] = _toolset_exists, + toolset_exists: Callable[[str], Awaitable[bool]] = saved_toolset_exists, ) -> frozenset[str]: registered: Final = tuple(servers()) if names else () return frozenset([name for name in names if _registry_serves(name, registered) or await toolset_exists(name)]) diff --git a/litellm/utils.py b/litellm/utils.py index fea6b9d2177..8cc0cd3ccce 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -438,11 +438,11 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy, reset_retry_policy, ) - from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index a46c8bed4ad..716c0af236c 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -18,7 +18,6 @@ from litellm.types.utils import ( ChatCompletionMessageToolCall, Delta, Function, - GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index ab38f4fc41f..d4775173fe0 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -952,3 +952,33 @@ def test_deepseek_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(mo assert "tools" not in result assert "tool_choice" not in result assert "parallel_tool_calls" not in result + + +def test_deepseek_clears_a_tool_choice_naming_the_hosted_mcp_tool_drop_params_dropped(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = DeepSeekChatConfig().map_openai_params( + { + "tools": [_HOSTED_MCP_TOOL, _function_tool("shell")], + "tool_choice": {"type": "function", "function": {"name": "deepwiki"}}, + }, + {}, + "deepseek-v4-flash", + True, + ) + assert result["tools"] == [_function_tool("shell")] + assert "tool_choice" not in result + + +def test_deepseek_removes_tool_selection_beside_legacy_functions_when_drop_params_drops_every_tool(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + functions: Final = [{"name": "shell", "parameters": {"type": "object", "properties": {}}}] + result: Final = DeepSeekChatConfig().map_openai_params( + {"tools": [_HOSTED_MCP_TOOL], "functions": functions, "tool_choice": "required", "parallel_tool_calls": True}, + {}, + "deepseek-v4-flash", + True, + ) + assert result["functions"] == functions + assert "tools" not in result + assert "tool_choice" not in result + assert "parallel_tool_calls" not in result diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py index 6501526634c..eeaa105565e 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py @@ -255,3 +255,30 @@ async def test_allows_saved_toolset_by_name_and_blocks_unknown_name(guardrail, c ): await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) assert exc.value.detail["unregistered_servers"] == ["evil_server"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) +async def test_blocks_the_name_with_the_policy_result_when_the_toolset_read_fails(guardrail, call_type): + prisma: Final = MagicMock() + prisma.writer_db = prisma.db + prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcptoolsettable.find_first = AsyncMock(side_effect=RuntimeError("toolset table unavailable")) + data: Final = { + "tools": [{"type": "mcp", "server_url": "litellm_proxy/mcp/research"}], + "guardrails": ["test-mcp-security"], + } + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + MCPServerManager(), + ), + pytest.raises(HTTPException) as exc, + ): + await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) + assert exc.value.status_code == 400 + assert exc.value.detail["unregistered_servers"] == ["research"]