From e25a82c3f54ff54178b5ebb98fb1da6a753e1fe2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:53:56 -0800 Subject: [PATCH] fix(tool-choice): refuse an excluded tool right before LiteLLM runs it Checking the provider up front missed a model alias, an old Azure api_version, a litellm_proxy hop, and SDK fallbacks. The tool loop and the MCP gateway now check the allowed_tools list right before they run a tool call, so an excluded call answers 400 and never runs. Under drop_params the choice now drops only when the caller runs every tool, so a provider-run MCP server or built-in keeps the 400. The Azure checks read the request's api_version, which also fixes the /v1/responses echo there. --- .../allowed_tools_tool_choice.py | 89 ++++++++-- litellm/llms/azure/chat/gpt_transformation.py | 7 +- litellm/main.py | 5 +- .../handler.py | 17 +- litellm/responses/main.py | 8 +- .../responses/mcp/chat_completions_handler.py | 2 + .../mcp/litellm_proxy_mcp_handler.py | 7 + .../responses/mcp/mcp_streaming_iterator.py | 1 + litellm/tool_loop.py | 44 ++--- .../test_allowed_tools_tool_choice.py | 69 ++++++++ .../test_anthropic_responses_bridge.py | 50 ++++++ .../mcp/test_aresponses_api_with_mcp.py | 103 ++++++++++-- .../mcp/test_chat_completions_handler.py | 94 ++++++++++- tests/unit/test_main.py | 16 ++ tests/unit/test_tool_loop.py | 155 ++++++++++++++---- 15 files changed, 569 insertions(+), 98 deletions(-) diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 36d110b212e..ff067b8b8e4 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Final, Literal from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam @@ -9,6 +9,8 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) +_FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE: Final = "2023-12-01" +_TOOL_TYPES_THE_CALLER_RUNS: Final = frozenset({"function", "custom"}) CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' @@ -57,11 +59,19 @@ class _ToolChoiceType(BaseModel): _TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType) -def is_allowed_tools_tool_choice(tool_choice: object) -> bool: +def _declared_type(value: object) -> str | None: try: - return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools" + return _TOOL_CHOICE_TYPE.validate_python(value).type except ValidationError: - return False + return None + + +def is_allowed_tools_tool_choice(tool_choice: object) -> bool: + return _declared_type(tool_choice) == "allowed_tools" + + +def caller_runs_every_tool(tools: Iterable[object] | None) -> bool: + return all(_declared_type(tool) in _TOOL_TYPES_THE_CALLER_RUNS for tool in tools or ()) def drops_unsupported_allowed_tools_tool_choice( @@ -69,11 +79,14 @@ def drops_unsupported_allowed_tools_tool_choice( custom_llm_provider: str, model: str, drop_params: bool, + tools: Iterable[object] | None, ) -> bool: if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: return False - if drop_params: + if drop_params and caller_runs_every_tool(tools): return True + if drop_params: + raise _a_choice_drop_params_keeps(custom_llm_provider, model) raise BadRequestError( message=( "tool_choice type 'allowed_tools' is only supported for " @@ -124,25 +137,75 @@ def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsTool return _chat_layout(parsed) -def unenforced_allowed_tools_tool_choice(tool_choice: object, custom_llm_provider: str | None) -> bool: +def azure_api_version_takes_tool_choice(api_version: str) -> bool: + return api_version.count("-") < 2 or api_version >= _FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE + + +def _azure_request_takes_tool_choice(api_version: str | None) -> bool: + import litellm + from litellm.secret_managers.main import get_secret_str + + return azure_api_version_takes_tool_choice( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION + ) + + +def _enforces_allowed_tools(custom_llm_provider: str | None, api_version: str | None) -> bool: + if custom_llm_provider == "azure": + return _azure_request_takes_tool_choice(api_version) + return custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS + + +def unenforced_allowed_tools_tool_choice( + tool_choice: object, custom_llm_provider: str | None, api_version: str | None = None +) -> bool: return is_allowed_tools_tool_choice(tool_choice) and ( - custom_llm_provider not in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS + not _enforces_allowed_tools(custom_llm_provider, api_version) or parse_allowed_tools_tool_choice(tool_choice) is None ) +def _a_choice_drop_params_keeps(custom_llm_provider: str | None, model: str) -> BadRequestError: + return BadRequestError( + message=( + f"This allowed_tools tool_choice can't be enforced on custom_llm_provider={custom_llm_provider}, and the " + "caller does not run every tool on this request, so drop_params does not drop it. Send only the allowed " + "tools in `tools` with tool_choice='auto' or 'required'." + ), + model=model, + llm_provider=custom_llm_provider or "", + ) + + def refuse_unenforced_allowed_tools_tool_choice( - tool_choice: object, custom_llm_provider: str | None, model: str + tool_choice: object, custom_llm_provider: str | None, model: str, api_version: str | None = None ) -> None: - if unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider): + if unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider, api_version): + raise _a_choice_drop_params_keeps(custom_llm_provider, model) + + +def _allowed_tool_names(tool_choice: object) -> frozenset[object]: + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) + if parsed is None: + return frozenset() + flattened: Final = tuple(_responses_allowed_tool(tool) for tool in parsed.allowed_tools.tools) + return frozenset(tool.get("name") for tool in flattened if tool is not None) + + +def refuse_tool_calls_the_choice_excludes(tool_choice: object, tool_names: Iterable[str], model: str) -> None: + if not is_allowed_tools_tool_choice(tool_choice): + return + allowed: Final = _allowed_tool_names(tool_choice) + excluded: Final = tuple(name for name in tool_names if name not in allowed) + if excluded: raise BadRequestError( message=( - f"This allowed_tools tool_choice can't be enforced on custom_llm_provider={custom_llm_provider}, and " - "LiteLLM runs the tool calls on this request, so drop_params does not drop it. Send only the allowed " - "tools in `tools` with tool_choice='auto' or 'required'." + f"The model called {', '.join(excluded)}, which the allowed_tools tool_choice excludes, because this " + "request did not reach a model that enforces the choice. LiteLLM did not run the call. Send only the " + "allowed tools in `tools` with tool_choice='auto' or 'required'." ), model=model, - llm_provider=custom_llm_provider or "", + llm_provider="", ) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index e0aceba52f9..7ca4ba6c163 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -6,6 +6,7 @@ from httpx._models import Headers, Response import litellm from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support +from litellm.litellm_core_utils.allowed_tools_tool_choice import azure_api_version_takes_tool_choice from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( chat_completion_prompt_cache_breakpoints_for_wire, @@ -240,11 +241,7 @@ class AzureOpenAIConfig(BaseConfig): if api_version_year is None or api_version_month is None or api_version_day is None: optional_params["tool_choice"] = value else: - if ( - api_version_year < "2023" - or (api_version_year == "2023" and api_version_month < "12") - or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01") - ): + if not azure_api_version_takes_tool_choice(api_version): if not (litellm.drop_params is True or drop_params is True): raise UnsupportedParamsError( status_code=400, diff --git a/litellm/main.py b/litellm/main.py index 0b95c3bd15c..b45a4ba8812 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -73,6 +73,7 @@ from litellm.constants import ( from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + caller_runs_every_tool, drops_unsupported_allowed_tools_tool_choice, refuse_unenforced_allowed_tools_tool_choice, ) @@ -5591,7 +5592,7 @@ def completion( drop_params=drops_unsupported_params( cast("Mapping[str, object]", kwargs).get("drop_params") # cast-ok: kwargs is the untyped request dict ) - and not may_name_litellm_mcp_gateway(tools), # pyright: ignore[reportUnknownArgumentType] # untyped tools list + and caller_runs_every_tool(tools), # pyright: ignore[reportUnknownArgumentType] # untyped tools list ) # validate optional params stop = validate_openai_optional_params(stop=stop) @@ -5896,6 +5897,7 @@ def completion( ), custom_llm_provider=custom_llm_provider, model=model, + api_version=api_version, ) drops_tool_choice: Final = drops_unsupported_allowed_tools_tool_choice( @@ -5905,6 +5907,7 @@ def completion( custom_llm_provider=custom_llm_provider, model=model, drop_params=drop_unsupported_reasoning, + tools=tools, # pyright: ignore[reportUnknownArgumentType] # untyped tools list ) if provider_specific_header is not None: diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 153b83c6aab..aa5c7028240 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -74,12 +74,17 @@ def _bridged_request( def _with_the_tool_choice_sent( - responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None + responses_api_request: ResponsesAPIOptionalRequestParams, + custom_llm_provider: str | None, + request_kwargs: Mapping[str, object], ) -> ResponsesAPIOptionalRequestParams: + api_version: Final = request_kwargs.get("api_version") tool_choice: Final[object] = responses_api_request.get( # pyright: ignore[reportUnknownMemberType] # ToolChoice has an untyped member "tool_choice" ) - if not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider): + if not unenforced_allowed_tools_tool_choice( + tool_choice, custom_llm_provider, api_version if isinstance(api_version, str) else None + ): return responses_api_request return {**responses_api_request, "tool_choice": None} @@ -139,7 +144,9 @@ class LiteLLMCompletionTransformationHandler: ), **_chat_verbosity(bridged_request), } - echoed_request: Final = _with_the_tool_choice_sent(bridged_request, custom_llm_provider) + echoed_request: Final = _with_the_tool_choice_sent( + bridged_request, custom_llm_provider, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ) if _is_async: return self.async_response_api_handler( @@ -222,7 +229,9 @@ class LiteLLMCompletionTransformationHandler: **_chat_verbosity(bridged.responses_api_request), }, request_input=bridged.input, - responses_api_request=_with_the_tool_choice_sent(bridged.responses_api_request, custom_llm_provider), + responses_api_request=_with_the_tool_choice_sent( + bridged.responses_api_request, custom_llm_provider, request_kwargs + ), **request_kwargs, ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5f671f7be08..3dbcc299008 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -478,6 +478,7 @@ async def aresponses_api_with_mcp( tool_server_map=tool_server_map, served_tools=original_mcp_tools, tool_calls=tool_calls, + tool_choice=tool_choice, user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, @@ -676,10 +677,12 @@ def _bridge_kwargs( def _refuse_a_choice_the_mcp_tool_run_would_ignore( - skip_mcp_handler: bool, tool_choice: object, custom_llm_provider: str | None, model: str + skip_mcp_handler: bool, tool_choice: object, custom_llm_provider: str | None, model: str, api_version: object ) -> None: if skip_mcp_handler: - refuse_unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider, model) + refuse_unenforced_allowed_tools_tool_choice( + tool_choice, custom_llm_provider, model, api_version if isinstance(api_version, str) else None + ) def _raise_untranslatable_responses_fields( @@ -1775,6 +1778,7 @@ def responses( tool_choice, custom_llm_provider, model, + kwargs.get("api_version"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped kwargs ) return litellm_completion_transformation_handler.response_api_handler( previous_response_client_ids(previous_response_id, kwargs), diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index f8a52227d1c..c716864d155 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -500,6 +500,7 @@ async def acompletion_with_mcp( tool_server_map=self.tool_server_map, served_tools=deduplicated_mcp_tools, tool_calls=self.tool_calls, + tool_choice=kwargs.get("tool_choice"), user_api_key_auth=self.user_api_key_auth, mcp_auth_header=self.mcp_auth_header, mcp_server_auth_headers=self.mcp_server_auth_headers, @@ -683,6 +684,7 @@ async def acompletion_with_mcp( tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( tool_server_map=tool_server_map, tool_calls=tool_calls, + tool_choice=kwargs.get("tool_choice"), served_tools=deduplicated_mcp_tools, user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 1104aee353e..d37ee58eb8f 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -10,6 +10,7 @@ from typing_extensions import Unpack from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG +from litellm.litellm_core_utils.allowed_tools_tool_choice import refuse_tool_calls_the_choice_excludes from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager from litellm.proxy.mcp_server.utils import ( @@ -740,6 +741,7 @@ class LiteLLM_Proxy_MCP_Handler: guardrail_context: Mapping[str, object] | None = None, served_tools: Sequence[MCPTool] | None = None, request_trace_id: str | None = None, + tool_choice: object = None, ) -> list[MCPToolResult]: """Execute tool calls and return results.""" from fastapi import HTTPException @@ -754,6 +756,11 @@ class LiteLLM_Proxy_MCP_Handler: ) from litellm.proxy.proxy_server import proxy_logging_obj + refuse_tool_calls_the_choice_excludes( + tool_choice, + tuple(LiteLLM_Proxy_MCP_Handler.extract_tool_call_details(tool_call)[0] or "" for tool_call in tool_calls), + "", + ) tool_results: Final[list[MCPToolResult]] = [] tool_call_id: str | None = None rules_obj: Final = Rules() diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 189366feab4..b5e3eee9ecf 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -898,6 +898,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): tool_server_map=self.tool_server_map, served_tools=self.served_tools, tool_calls=tool_calls, + tool_choice=self.original_request_params.get("tool_choice"), user_api_key_auth=self.user_api_key_auth, mcp_auth_header=self.mcp_auth_header, mcp_server_auth_headers=self.mcp_server_auth_headers, diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index 81d1a8f1b32..1db6b8c6000 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -6,10 +6,7 @@ from typing import Final, TypeAlias, cast from typing_extensions import TypedDict, Unpack from litellm.constants import DEFAULT_TOOL_LOOP_MAX_ROUNDS -from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - is_allowed_tools_tool_choice, - refuse_unenforced_allowed_tools_tool_choice, -) +from litellm.litellm_core_utils.allowed_tools_tool_choice import refuse_tool_calls_the_choice_excludes from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, @@ -36,32 +33,11 @@ class ToolLoopMaxRoundsExceeded(RuntimeError): super().__init__(f"model still requested tool calls on round {max_rounds} of {max_rounds}") -def _text(value: object) -> str | None: - return value if isinstance(value, str) else None - - -def _refuse_unenforced_tool_choice(model: str, completion_kwargs: _ToolLoopCompletionKwargs) -> None: - import litellm - - tool_choice: Final = completion_kwargs.get("tool_choice") - if is_allowed_tools_tool_choice(tool_choice): - refuse_unenforced_allowed_tools_tool_choice( - tool_choice, - litellm.get_llm_provider( - model=model, - custom_llm_provider=_text(completion_kwargs.get("custom_llm_provider")), - api_base=_text(completion_kwargs.get("api_base")), - )[1], - model, - ) - - -def _validate_tool_loop_args(model: str, max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None: +def _validate_tool_loop_args(max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None: if max_rounds < 1: raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") if completion_kwargs.get("stream"): raise ValueError("run_tool_loop requires whole responses; stream=True is not supported") - _refuse_unenforced_tool_choice(model, completion_kwargs) def _expect_model_response(response: object) -> ModelResponse: @@ -139,6 +115,16 @@ def _client_tool_calls( ) +def _refuse_tool_calls_the_choice_excludes( + tool_calls: tuple[ChatCompletionMessageToolCall, ...], model: str, completion_kwargs: _ToolLoopCompletionKwargs +) -> None: + refuse_tool_calls_the_choice_excludes( + completion_kwargs.get("tool_choice"), + tuple(tool_call.function.name or "" for tool_call in tool_calls), + model, + ) + + def _replays_provider_specific_fields( message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], @@ -173,7 +159,7 @@ def run_tool_loop( """ import litellm - _validate_tool_loop_args(model, max_rounds, completion_kwargs) + _validate_tool_loop_args(max_rounds, completion_kwargs) history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns for round_number in range(1, max_rounds + 1): message = ( @@ -191,6 +177,7 @@ def run_tool_loop( return message.content if round_number == max_rounds: break + _refuse_tool_calls_the_choice_excludes(client_tool_calls, model, completion_kwargs) tool_results = tuple(execute_tool(tool_call) for tool_call in client_tool_calls) history = ( *history, @@ -224,7 +211,7 @@ async def arun_tool_loop( """ import litellm - _validate_tool_loop_args(model, max_rounds, completion_kwargs) + _validate_tool_loop_args(max_rounds, completion_kwargs) history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns for round_number in range(1, max_rounds + 1): message = ( @@ -242,6 +229,7 @@ async def arun_tool_loop( return message.content if round_number == max_rounds: break + _refuse_tool_calls_the_choice_excludes(client_tool_calls, model, completion_kwargs) tool_results = tuple([await execute_tool(tool_call) for tool_call in client_tool_calls]) history = ( *history, diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index 97d4d6d5152..698c5dcf2f5 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -4,12 +4,15 @@ from typing import Final import pytest +import litellm from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ChatAllowedToolsToolChoice, + drops_unsupported_allowed_tools_tool_choice, is_allowed_tools_tool_choice, chat_allowed_tools_tool_choice, parse_allowed_tools_tool_choice, + refuse_tool_calls_the_choice_excludes, refuse_unenforced_allowed_tools_tool_choice, unenforced_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, @@ -184,3 +187,69 @@ def test_a_tool_runner_keeps_a_choice_the_provider_enforces_or_any_other_choice( tool_choice: object, custom_llm_provider: str ): assert not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider) + + +@pytest.mark.parametrize( + ("tool_choice", "tool_names"), + [(_FLAT_CHOICE, ("get_time",)), (_CHAT_LAYOUT, ("get_time", "apply_patch"))], +) +def test_a_tool_call_the_choice_lists_passes(tool_choice: object, tool_names: tuple[str, ...]): + assert refuse_tool_calls_the_choice_excludes(tool_choice, tool_names, "the-model") is None + + +@pytest.mark.parametrize( + "tool_choice", [_FLAT_CHOICE, _CHAT_LAYOUT, {"type": "allowed_tools", "mode": "required", "tools": []}] +) +def test_a_tool_call_the_choice_does_not_list_is_refused(tool_choice: object): + with pytest.raises(BadRequestError, match=r"The model called .*get_weather") as exc: + refuse_tool_calls_the_choice_excludes(tool_choice, ("get_time", "get_weather"), "the-model") + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("tool_choice", ["required", None, {"type": "function", "function": {"name": "get_time"}}]) +def test_any_other_choice_leaves_the_tool_calls_alone(tool_choice: object): + assert refuse_tool_calls_the_choice_excludes(tool_choice, ("get_weather",), "the-model") is None + + +@pytest.mark.parametrize( + ("api_version", "unenforced"), + [ + ("2023-07-01-preview", True), + ("2023-11-30", True), + ("2023-12-01-preview", False), + ("2025-04-01-preview", False), + ("v1", False), + ], +) +def test_azure_counts_as_enforcing_only_on_a_version_that_takes_tool_choice(api_version: str, unenforced: bool): + assert unenforced_allowed_tools_tool_choice(_FLAT_CHOICE, "azure", api_version) is unenforced + + +def test_azure_reads_the_same_api_version_fallback_as_the_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "api_version", "2023-07-01-preview") + + assert unenforced_allowed_tools_tool_choice(_FLAT_CHOICE, "azure") + + +_FUNCTION_TOOL: Final = {"type": "function", "function": {"name": "get_time"}} + + +@pytest.mark.parametrize( + "tools", + [ + [_FUNCTION_TOOL, {"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com"}], + [{"type": "web_search_20250305", "name": "web_search"}], + [{"googleSearch": {}}], + [{"type": ["function"]}], + ["get_time"], + ], +) +def test_drop_params_keeps_the_400_when_the_provider_could_run_a_tool(tools: list[object]): + with pytest.raises(BadRequestError, match="drop_params does not drop it"): + drops_unsupported_allowed_tools_tool_choice(_FLAT_CHOICE, "anthropic", "the-model", True, tools) + + +@pytest.mark.parametrize("tools", [None, [], [_FUNCTION_TOOL, {"type": "custom", "custom": {"name": "patch"}}]]) +def test_drop_params_drops_the_choice_when_the_caller_runs_every_tool(tools: list[object] | None): + assert drops_unsupported_allowed_tools_tool_choice(_FLAT_CHOICE, "anthropic", "the-model", True, tools) diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 66b13b3bbd3..79a655fc913 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -2132,6 +2132,56 @@ async def test_responses_reports_auto_for_an_allowed_tools_choice_it_dropped( assert response.tool_choice == "auto" +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.asyncio +async def test_responses_keep_the_400_when_the_provider_would_run_an_mcp_tool(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"): + await litellm.aresponses( + model="anthropic/claude-haiku-4-5", + input="What is the weather?", + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}], + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + api_key="sk-ant-fake", + ) + + assert not respx_mock.calls + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_responses_report_auto_when_an_old_azure_api_version_dropped_the_choice(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(url__regex=r"https://azure-bridge\.example\.com/.*chat/completions.*").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-old-azure", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + response: Final = litellm.responses( + model="azure/gpt-4o", + input="What is the weather?", + tools=list(_RESPONSES_TOOLS), + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + use_chat_completions_api=True, + api_key="fake", + api_base="https://azure-bridge.example.com", + api_version="2023-07-01-preview", + ) + + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert response.tool_choice == "auto" + + def test_streamed_responses_report_auto_for_an_allowed_tools_choice_it_dropped(respx_mock: respx.MockRouter) -> None: respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( return_value=httpx.Response( diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index f8eb0378ed1..6d6546b1d7e 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -5,7 +5,7 @@ import logging import sys import warnings from collections.abc import Callable, Iterator, Mapping, Sequence -from contextlib import contextmanager, nullcontext +from contextlib import AbstractContextManager, contextmanager, nullcontext from datetime import datetime from types import MappingProxyType, SimpleNamespace from typing import Any, Final, List, cast @@ -1727,9 +1727,7 @@ async def _auto_execute_echo( return await _auto_execute(logging_obj, _ECHO_MCP_TOOL, "PING") -async def _auto_execute_through_mcp_server( - monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock, logging_obj: Logging -) -> None: +def _served_by_a_recording_mcp_server(monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock) -> None: monkeypatch.setitem( sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) ) @@ -1743,15 +1741,31 @@ async def _auto_execute_through_mcp_server( get_mcp_server_by_name=MagicMock(return_value=None), ), ) - echo_tool: Final = MCPTool.model_validate( - {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, - by_name=False, - ) - with patch.object( + + +def _echo_tool_listed() -> AbstractContextManager[object]: + return patch.object( LiteLLM_Proxy_MCP_Handler, "process_mcp_tools_without_openai_transform", - AsyncMock(return_value=([echo_tool], {"echo": "demo"})), - ): + AsyncMock( + return_value=( + [ + MCPTool.model_validate( + {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, + by_name=False, + ) + ], + {"echo": "demo"}, + ) + ), + ) + + +async def _auto_execute_through_mcp_server( + monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock, logging_obj: Logging +) -> None: + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + with _echo_tool_listed(): await litellm.aresponses( model="openai/gpt-5.6", input="Call echo with PING", @@ -1763,6 +1777,73 @@ async def _auto_execute_through_mcp_server( ) +_ALLOWED_ONLY_ANOTHER_TOOL: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "another_tool"}], +} + + +async def _call_with_a_choice_excluding_echo(stream: bool) -> tuple[object, ...]: + with _echo_tool_listed(): + response: Final = await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + tool_choice=_ALLOWED_ONLY_ANOTHER_TOOL, + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + stream=stream, + ) + return await _drain(response) if stream else (response,) + + +@pytest.mark.asyncio +async def test_an_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + _serve_rounds(respx_mock, monkeypatch, (_ROUND_ONE_BODY,)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): + await _call_with_a_choice_excluding_echo(stream=False) + + call_tool.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_an_mcp_tool_the_choice_lists_still_runs(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + _serve_rounds(respx_mock, monkeypatch, (_ROUND_ONE_BODY, _ROUND_TWO_BODY)) + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + with _echo_tool_listed(): + await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "echo"}]}, + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + ) + + assert call_tool.await_count == 1 + + +@pytest.mark.asyncio +async def test_a_streamed_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY),)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + await _call_with_a_choice_excluding_echo(stream=True) + + call_tool.assert_not_awaited() + + def _rounds_logging_obj(recorder: _RoundRecorder, call_id: str) -> Logging: return Logging( model="openai/gpt-5.6", diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index ecef6b44bae..c771b41ff64 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -7,9 +7,9 @@ import threading import types import warnings from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from datetime import datetime -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -2782,3 +2782,93 @@ async def test_mcp_tools_keep_the_400_for_an_allowed_tools_choice_the_provider_c execute.assert_not_awaited() assert not respx_mock.calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_mcp_tools_keep_the_400_on_an_azure_api_version_that_drops_the_choice( + respx_mock: respx.MockRouter, stream: bool +): + execute: Final = AsyncMock(return_value=[]) + + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([_MCP_ECHO_TOOL], {"echo": "demo"})), + ), + patch.object(LiteLLM_Proxy_MCP_Handler, "execute_tool_calls", execute), + pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"), + ): + await litellm.acompletion( + model="azure/gpt-4o", + messages=[{"role": "user", "content": "Call echo"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice=_FLAT_ALLOWED_ECHO, + drop_params=True, + stream=stream, + api_key="fake", + api_base="https://mcp-chat-rounds.example.com", + api_version="2023-07-01-preview", + ) + + execute.assert_not_awaited() + assert not respx_mock.calls + + +_ALLOWED_ONLY_ANOTHER_TOOL: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "another_tool"}], +} + + +def _served_by_a_recording_mcp_server(monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock) -> None: + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) + ) + monkeypatch.setattr( + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", + SimpleNamespace( + catalog=SimpleNamespace(operation=nullcontext), + get_registry=MagicMock(return_value={}), + call_tool=call_tool, + get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_an_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE) if stream else _ok(_CHAT_ROUND_ONE),)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + async def call_with_the_choice() -> None: + response: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice=_ALLOWED_ONLY_ANOTHER_TOOL, + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + stream=stream, + ) + if stream: + await _read_all(response) + + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([_MCP_ECHO_TOOL], {"echo": "demo"})), + ), + pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"), + ): + await call_with_the_choice() + + call_tool.assert_not_awaited() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 4f60440ff68..263cd376a8a 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -664,6 +664,22 @@ async def test_a_malformed_allowed_tools_tool_choice_stays_a_400_on_mcp_gateway_ assert not respx_mock.calls +@pytest.mark.asyncio +async def test_drop_params_keeps_the_400_when_the_provider_would_run_an_mcp_tool(respx_mock: respx.MockRouter): + with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it") as exc: + await litellm.acompletion( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "What time is it?"}], + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + drop_params=True, + api_key="sk-ant-fake", + ) + + assert exc.value.status_code == 400 + assert not respx_mock.calls + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", diff --git a/tests/unit/test_tool_loop.py b/tests/unit/test_tool_loop.py index 22bfd4ccf1d..9b724b644b6 100644 --- a/tests/unit/test_tool_loop.py +++ b/tests/unit/test_tool_loop.py @@ -11,6 +11,7 @@ from litellm.types.llms.openai import ChatCompletionToolMessage from litellm.types.utils import ChatCompletionMessageToolCall OPENAI_CHAT_COMPLETIONS_URL: Final = "https://api.openai.com/v1/chat/completions" +ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages" WEATHER_TOOLS: Final = ( { "type": "function", @@ -248,57 +249,147 @@ def test_stream_rejected_before_any_request(respx_mock: respx.MockRouter) -> Non assert route.call_count == 0 -_FLAT_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "a"}]} +_ALLOWED_WEATHER_ONLY: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_weather"}], +} _EMPTY_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": []} +_TIME_CALL: Final = _tool_call("call_1", "get_time", {}) -@pytest.mark.parametrize( - ("model", "tool_choice"), - [("anthropic/claude-haiku-4-5", _FLAT_ALLOWED_TOOLS), ("openai/gpt-5-mini", _EMPTY_ALLOWED_TOOLS)], -) -def test_allowed_tools_choice_nothing_enforces_is_refused_under_drop_params( - respx_mock: respx.MockRouter, model: str, tool_choice: dict[str, object] -) -> None: +def _anthropic_message(*content: dict[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_tool_loop", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": list(content), + "stop_reason": "tool_use" if content[0]["type"] == "tool_use" else "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +def _anthropic_tool_use(name: str) -> httpx.Response: + return _anthropic_message({"type": "tool_use", "id": "toolu_1", "name": name, "input": {}}) + + +def _tools_run_before_the_refusal( + tool_choice: dict[str, object], **completion_kwargs: object +) -> list[ChatCompletionMessageToolCall]: executed: Final[list[ChatCompletionMessageToolCall]] = [] def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: executed.append(tool_call) return _tool_result(tool_call) - with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"): + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): litellm.run_tool_loop( - model=model, messages=[{"role": "user", "content": "hi"}], tools=WEATHER_TOOLS, execute_tool=execute, tool_choice=tool_choice, + api_key="sk-test", + **completion_kwargs, + ) + return executed + + +def test_a_tool_the_dropped_choice_excludes_never_runs(respx_mock: respx.MockRouter) -> None: + respx_mock.post(ANTHROPIC_MESSAGES_URL).mock(return_value=_anthropic_tool_use("get_time")) + + assert not _tools_run_before_the_refusal( + _ALLOWED_WEATHER_ONLY, model="anthropic/claude-haiku-4-5", drop_params=True + ) + assert "tool_choice" not in _request_bodies(respx_mock)[0] + + +def test_a_tool_the_dropped_choice_lists_still_runs(respx_mock: respx.MockRouter) -> None: + respx_mock.post(ANTHROPIC_MESSAGES_URL).mock( + side_effect=[_anthropic_tool_use("get_weather"), _anthropic_message({"type": "text", "text": "done"})] + ) + + assert ( + litellm.run_tool_loop( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=_ALLOWED_WEATHER_ONLY, + drop_params=True, + api_key="sk-test", + ) + == "done" + ) + + +@pytest.mark.parametrize( + ("tool_choice", "completion_kwargs"), + [(_ALLOWED_WEATHER_ONLY, {}), (_EMPTY_ALLOWED_TOOLS, {"drop_params": True})], +) +def test_a_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, tool_choice: dict[str, object], completion_kwargs: dict[str, object] +) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [_TIME_CALL])) + ) + + assert not _tools_run_before_the_refusal(tool_choice, model="openai/gpt-5-mini", **completion_kwargs) + + +def test_a_choice_on_a_model_alias_reaches_the_loop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "model_alias_map", {"the-alias": "openai/gpt-5-mini"}) + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response("done")) + ) + + assert ( + litellm.run_tool_loop( + model="the-alias", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=_ALLOWED_WEATHER_ONLY, + api_key="sk-test", + ) + == "done" + ) + + +@pytest.mark.asyncio +async def test_async_loop_never_runs_a_tool_the_choice_excludes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler()) + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [_TIME_CALL])) + ) + executed: Final[list[ChatCompletionMessageToolCall]] = [] + + async def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tool_call) + return _tool_result(tool_call) + + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): + await litellm.arun_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=execute, + tool_choice=_ALLOWED_WEATHER_ONLY, drop_params=True, api_key="sk-test", ) assert not executed - assert not respx_mock.calls - - -@pytest.mark.asyncio -async def test_async_loop_refuses_an_allowed_tools_choice_the_provider_cannot_enforce( - respx_mock: respx.MockRouter, -) -> None: - async def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: - return _tool_result(tool_call) - - with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"): - await litellm.arun_tool_loop( - model="anthropic/claude-haiku-4-5", - messages=[{"role": "user", "content": "hi"}], - tools=WEATHER_TOOLS, - execute_tool=execute, - tool_choice=_FLAT_ALLOWED_TOOLS, - drop_params=True, - api_key="sk-test", - ) - - assert not respx_mock.calls def test_allowed_tools_choice_the_provider_enforces_reaches_it_every_round(respx_mock: respx.MockRouter) -> None: