diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 36d110b212e..ff067b8b8e4 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Final, Literal from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam @@ -9,6 +9,8 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) +_FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE: Final = "2023-12-01" +_TOOL_TYPES_THE_CALLER_RUNS: Final = frozenset({"function", "custom"}) CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' @@ -57,11 +59,19 @@ class _ToolChoiceType(BaseModel): _TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType) -def is_allowed_tools_tool_choice(tool_choice: object) -> bool: +def _declared_type(value: object) -> str | None: try: - return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools" + return _TOOL_CHOICE_TYPE.validate_python(value).type except ValidationError: - return False + return None + + +def is_allowed_tools_tool_choice(tool_choice: object) -> bool: + return _declared_type(tool_choice) == "allowed_tools" + + +def caller_runs_every_tool(tools: Iterable[object] | None) -> bool: + return all(_declared_type(tool) in _TOOL_TYPES_THE_CALLER_RUNS for tool in tools or ()) def drops_unsupported_allowed_tools_tool_choice( @@ -69,11 +79,14 @@ def drops_unsupported_allowed_tools_tool_choice( custom_llm_provider: str, model: str, drop_params: bool, + tools: Iterable[object] | None, ) -> bool: if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: return False - if drop_params: + if drop_params and caller_runs_every_tool(tools): return True + if drop_params: + raise _a_choice_drop_params_keeps(custom_llm_provider, model) raise BadRequestError( message=( "tool_choice type 'allowed_tools' is only supported for " @@ -124,25 +137,75 @@ def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsTool return _chat_layout(parsed) -def unenforced_allowed_tools_tool_choice(tool_choice: object, custom_llm_provider: str | None) -> bool: +def azure_api_version_takes_tool_choice(api_version: str) -> bool: + return api_version.count("-") < 2 or api_version >= _FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE + + +def _azure_request_takes_tool_choice(api_version: str | None) -> bool: + import litellm + from litellm.secret_managers.main import get_secret_str + + return azure_api_version_takes_tool_choice( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION + ) + + +def _enforces_allowed_tools(custom_llm_provider: str | None, api_version: str | None) -> bool: + if custom_llm_provider == "azure": + return _azure_request_takes_tool_choice(api_version) + return custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS + + +def unenforced_allowed_tools_tool_choice( + tool_choice: object, custom_llm_provider: str | None, api_version: str | None = None +) -> bool: return is_allowed_tools_tool_choice(tool_choice) and ( - custom_llm_provider not in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS + not _enforces_allowed_tools(custom_llm_provider, api_version) or parse_allowed_tools_tool_choice(tool_choice) is None ) +def _a_choice_drop_params_keeps(custom_llm_provider: str | None, model: str) -> BadRequestError: + return BadRequestError( + message=( + f"This allowed_tools tool_choice can't be enforced on custom_llm_provider={custom_llm_provider}, and the " + "caller does not run every tool on this request, so drop_params does not drop it. Send only the allowed " + "tools in `tools` with tool_choice='auto' or 'required'." + ), + model=model, + llm_provider=custom_llm_provider or "", + ) + + def refuse_unenforced_allowed_tools_tool_choice( - tool_choice: object, custom_llm_provider: str | None, model: str + tool_choice: object, custom_llm_provider: str | None, model: str, api_version: str | None = None ) -> None: - if unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider): + if unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider, api_version): + raise _a_choice_drop_params_keeps(custom_llm_provider, model) + + +def _allowed_tool_names(tool_choice: object) -> frozenset[object]: + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) + if parsed is None: + return frozenset() + flattened: Final = tuple(_responses_allowed_tool(tool) for tool in parsed.allowed_tools.tools) + return frozenset(tool.get("name") for tool in flattened if tool is not None) + + +def refuse_tool_calls_the_choice_excludes(tool_choice: object, tool_names: Iterable[str], model: str) -> None: + if not is_allowed_tools_tool_choice(tool_choice): + return + allowed: Final = _allowed_tool_names(tool_choice) + excluded: Final = tuple(name for name in tool_names if name not in allowed) + if excluded: raise BadRequestError( message=( - f"This allowed_tools tool_choice can't be enforced on custom_llm_provider={custom_llm_provider}, and " - "LiteLLM runs the tool calls on this request, so drop_params does not drop it. Send only the allowed " - "tools in `tools` with tool_choice='auto' or 'required'." + f"The model called {', '.join(excluded)}, which the allowed_tools tool_choice excludes, because this " + "request did not reach a model that enforces the choice. LiteLLM did not run the call. Send only the " + "allowed tools in `tools` with tool_choice='auto' or 'required'." ), model=model, - llm_provider=custom_llm_provider or "", + llm_provider="", ) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 56edee1713b..50507d24f16 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Final from httpx._models import Headers, Response import litellm +from litellm.litellm_core_utils.allowed_tools_tool_choice import azure_api_version_takes_tool_choice from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, flatten_combinators_and_drop_non_python_regex_patterns, @@ -219,11 +220,7 @@ class AzureOpenAIConfig(BaseConfig): if api_version_year is None or api_version_month is None or api_version_day is None: optional_params["tool_choice"] = value else: - if ( - api_version_year < "2023" - or (api_version_year == "2023" and api_version_month < "12") - or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01") - ): + if not azure_api_version_takes_tool_choice(api_version): if not (litellm.drop_params is True or drop_params is True): raise UnsupportedParamsError( status_code=400, diff --git a/litellm/main.py b/litellm/main.py index 5e799b8c742..fec4ef3b05e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -70,6 +70,7 @@ from litellm.constants import ( from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + caller_runs_every_tool, drops_unsupported_allowed_tools_tool_choice, refuse_unenforced_allowed_tools_tool_choice, ) @@ -5361,7 +5362,7 @@ def completion( tool_choice=tool_choice, model=model, drop_params=drops_unsupported - and not may_name_litellm_mcp_gateway(tools), # pyright: ignore[reportUnknownArgumentType] # untyped tools list + and caller_runs_every_tool(tools), # pyright: ignore[reportUnknownArgumentType] # untyped tools list ) # validate optional params stop = validate_openai_optional_params(stop=stop) @@ -5626,6 +5627,7 @@ def completion( ), custom_llm_provider=custom_llm_provider, model=model, + api_version=api_version, ) drops_tool_choice: Final = drops_unsupported_allowed_tools_tool_choice( @@ -5635,6 +5637,7 @@ def completion( custom_llm_provider=custom_llm_provider, model=model, drop_params=drops_unsupported, + tools=tools, # pyright: ignore[reportUnknownArgumentType] # untyped tools list ) if provider_specific_header is not None: diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 58e76b6fdb0..62445e72787 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -28,12 +28,17 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) def _with_the_tool_choice_sent( - responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None + responses_api_request: ResponsesAPIOptionalRequestParams, + custom_llm_provider: str | None, + request_kwargs: Mapping[str, object], ) -> ResponsesAPIOptionalRequestParams: + api_version: Final = request_kwargs.get("api_version") tool_choice: Final[object] = responses_api_request.get( # pyright: ignore[reportUnknownMemberType] # ToolChoice has an untyped member "tool_choice" ) - if not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider): + if not unenforced_allowed_tools_tool_choice( + tool_choice, custom_llm_provider, api_version if isinstance(api_version, str) else None + ): return responses_api_request return {**responses_api_request, "tool_choice": None} @@ -70,7 +75,9 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) ) - echoed_request: Final = _with_the_tool_choice_sent(bridged_request, custom_llm_provider) + echoed_request: Final = _with_the_tool_choice_sent( + bridged_request, custom_llm_provider, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ) if _is_async: return self.async_response_api_handler( diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 503cb12ee0f..4ef267774f9 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -348,6 +348,7 @@ async def aresponses_api_with_mcp( tool_server_map=tool_server_map, served_tools=original_mcp_tools, tool_calls=tool_calls, + tool_choice=tool_choice, user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, @@ -465,10 +466,12 @@ def _bridge_kwargs( def _refuse_a_choice_the_mcp_tool_run_would_ignore( - skip_mcp_handler: bool, tool_choice: object, custom_llm_provider: str | None, model: str + skip_mcp_handler: bool, tool_choice: object, custom_llm_provider: str | None, model: str, api_version: object ) -> None: if skip_mcp_handler: - refuse_unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider, model) + refuse_unenforced_allowed_tools_tool_choice( + tool_choice, custom_llm_provider, model, api_version if isinstance(api_version, str) else None + ) _ResponsesCompatibilityFailure: TypeAlias = Literal["encrypted_task_unsupported"] @@ -1511,6 +1514,7 @@ def responses( tool_choice, custom_llm_provider, model, + kwargs.get("api_version"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped kwargs ) return litellm_completion_transformation_handler.response_api_handler( model=model, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index a8c93eebff3..22c5c742b66 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -441,6 +441,7 @@ async def acompletion_with_mcp( tool_server_map=self.tool_server_map, served_tools=deduplicated_mcp_tools, tool_calls=self.tool_calls, + tool_choice=kwargs.get("tool_choice"), user_api_key_auth=self.user_api_key_auth, mcp_auth_header=self.mcp_auth_header, mcp_server_auth_headers=self.mcp_server_auth_headers, @@ -614,6 +615,7 @@ async def acompletion_with_mcp( tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( tool_server_map=tool_server_map, tool_calls=tool_calls, + tool_choice=kwargs.get("tool_choice"), served_tools=deduplicated_mcp_tools, user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 1dfa4b2abcb..e6b141e5774 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -11,6 +11,7 @@ from typing_extensions import Unpack from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG +from litellm.litellm_core_utils.allowed_tools_tool_choice import refuse_tool_calls_the_choice_excludes from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager from litellm.proxy._experimental.mcp_server.utils import ( @@ -729,6 +730,7 @@ class LiteLLM_Proxy_MCP_Handler: request_tags: list[str] | None = None, guardrail_context: Mapping[str, object] | None = None, served_tools: Sequence[MCPTool] | None = None, + tool_choice: object = None, ) -> list[MCPToolResult]: """Execute tool calls and return results.""" from fastapi import HTTPException @@ -743,6 +745,11 @@ class LiteLLM_Proxy_MCP_Handler: ) from litellm.proxy.proxy_server import proxy_logging_obj + refuse_tool_calls_the_choice_excludes( + tool_choice, + tuple(LiteLLM_Proxy_MCP_Handler.extract_tool_call_details(tool_call)[0] or "" for tool_call in tool_calls), + "", + ) tool_results: Final[list[MCPToolResult]] = [] tool_call_id: str | None = None rules_obj: Final = Rules() diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 73cfba8dca4..62c25f7367b 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -805,6 +805,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): tool_server_map=self.tool_server_map, served_tools=self.served_tools, tool_calls=tool_calls, + tool_choice=self.original_request_params.get("tool_choice"), user_api_key_auth=self.user_api_key_auth, mcp_auth_header=self.mcp_auth_header, mcp_server_auth_headers=self.mcp_server_auth_headers, diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index b0d29865900..2397b5692de 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -6,10 +6,7 @@ from typing import Final, TypeAlias, cast from typing_extensions import TypedDict, Unpack from litellm.constants import DEFAULT_TOOL_LOOP_MAX_ROUNDS -from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - is_allowed_tools_tool_choice, - refuse_unenforced_allowed_tools_tool_choice, -) +from litellm.litellm_core_utils.allowed_tools_tool_choice import refuse_tool_calls_the_choice_excludes from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, @@ -36,32 +33,11 @@ class ToolLoopMaxRoundsExceeded(RuntimeError): super().__init__(f"model still requested tool calls on round {max_rounds} of {max_rounds}") -def _text(value: object) -> str | None: - return value if isinstance(value, str) else None - - -def _refuse_unenforced_tool_choice(model: str, completion_kwargs: _ToolLoopCompletionKwargs) -> None: - import litellm - - tool_choice: Final = completion_kwargs.get("tool_choice") - if is_allowed_tools_tool_choice(tool_choice): - refuse_unenforced_allowed_tools_tool_choice( - tool_choice, - litellm.get_llm_provider( - model=model, - custom_llm_provider=_text(completion_kwargs.get("custom_llm_provider")), - api_base=_text(completion_kwargs.get("api_base")), - )[1], - model, - ) - - -def _validate_tool_loop_args(model: str, max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None: +def _validate_tool_loop_args(max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None: if max_rounds < 1: raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") if completion_kwargs.get("stream"): raise ValueError("run_tool_loop requires whole responses; stream=True is not supported") - _refuse_unenforced_tool_choice(model, completion_kwargs) def _expect_model_response(response: object) -> ModelResponse: @@ -137,6 +113,16 @@ def _client_tool_calls( ) +def _refuse_tool_calls_the_choice_excludes( + tool_calls: tuple[ChatCompletionMessageToolCall, ...], model: str, completion_kwargs: _ToolLoopCompletionKwargs +) -> None: + refuse_tool_calls_the_choice_excludes( + completion_kwargs.get("tool_choice"), + tuple(tool_call.function.name or "" for tool_call in tool_calls), + model, + ) + + def _replays_provider_specific_fields( message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], @@ -171,7 +157,7 @@ def run_tool_loop( """ import litellm - _validate_tool_loop_args(model, max_rounds, completion_kwargs) + _validate_tool_loop_args(max_rounds, completion_kwargs) history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns for round_number in range(1, max_rounds + 1): message = ( @@ -189,6 +175,7 @@ def run_tool_loop( return message.content if round_number == max_rounds: break + _refuse_tool_calls_the_choice_excludes(client_tool_calls, model, completion_kwargs) tool_results = tuple(execute_tool(tool_call) for tool_call in client_tool_calls) history = ( *history, @@ -222,7 +209,7 @@ async def arun_tool_loop( """ import litellm - _validate_tool_loop_args(model, max_rounds, completion_kwargs) + _validate_tool_loop_args(max_rounds, completion_kwargs) history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns for round_number in range(1, max_rounds + 1): message = ( @@ -240,6 +227,7 @@ async def arun_tool_loop( return message.content if round_number == max_rounds: break + _refuse_tool_calls_the_choice_excludes(client_tool_calls, model, completion_kwargs) tool_results = tuple([await execute_tool(tool_call) for tool_call in client_tool_calls]) history = ( *history, diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index 97d4d6d5152..698c5dcf2f5 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -4,12 +4,15 @@ from typing import Final import pytest +import litellm from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ChatAllowedToolsToolChoice, + drops_unsupported_allowed_tools_tool_choice, is_allowed_tools_tool_choice, chat_allowed_tools_tool_choice, parse_allowed_tools_tool_choice, + refuse_tool_calls_the_choice_excludes, refuse_unenforced_allowed_tools_tool_choice, unenforced_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, @@ -184,3 +187,69 @@ def test_a_tool_runner_keeps_a_choice_the_provider_enforces_or_any_other_choice( tool_choice: object, custom_llm_provider: str ): assert not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider) + + +@pytest.mark.parametrize( + ("tool_choice", "tool_names"), + [(_FLAT_CHOICE, ("get_time",)), (_CHAT_LAYOUT, ("get_time", "apply_patch"))], +) +def test_a_tool_call_the_choice_lists_passes(tool_choice: object, tool_names: tuple[str, ...]): + assert refuse_tool_calls_the_choice_excludes(tool_choice, tool_names, "the-model") is None + + +@pytest.mark.parametrize( + "tool_choice", [_FLAT_CHOICE, _CHAT_LAYOUT, {"type": "allowed_tools", "mode": "required", "tools": []}] +) +def test_a_tool_call_the_choice_does_not_list_is_refused(tool_choice: object): + with pytest.raises(BadRequestError, match=r"The model called .*get_weather") as exc: + refuse_tool_calls_the_choice_excludes(tool_choice, ("get_time", "get_weather"), "the-model") + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("tool_choice", ["required", None, {"type": "function", "function": {"name": "get_time"}}]) +def test_any_other_choice_leaves_the_tool_calls_alone(tool_choice: object): + assert refuse_tool_calls_the_choice_excludes(tool_choice, ("get_weather",), "the-model") is None + + +@pytest.mark.parametrize( + ("api_version", "unenforced"), + [ + ("2023-07-01-preview", True), + ("2023-11-30", True), + ("2023-12-01-preview", False), + ("2025-04-01-preview", False), + ("v1", False), + ], +) +def test_azure_counts_as_enforcing_only_on_a_version_that_takes_tool_choice(api_version: str, unenforced: bool): + assert unenforced_allowed_tools_tool_choice(_FLAT_CHOICE, "azure", api_version) is unenforced + + +def test_azure_reads_the_same_api_version_fallback_as_the_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "api_version", "2023-07-01-preview") + + assert unenforced_allowed_tools_tool_choice(_FLAT_CHOICE, "azure") + + +_FUNCTION_TOOL: Final = {"type": "function", "function": {"name": "get_time"}} + + +@pytest.mark.parametrize( + "tools", + [ + [_FUNCTION_TOOL, {"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com"}], + [{"type": "web_search_20250305", "name": "web_search"}], + [{"googleSearch": {}}], + [{"type": ["function"]}], + ["get_time"], + ], +) +def test_drop_params_keeps_the_400_when_the_provider_could_run_a_tool(tools: list[object]): + with pytest.raises(BadRequestError, match="drop_params does not drop it"): + drops_unsupported_allowed_tools_tool_choice(_FLAT_CHOICE, "anthropic", "the-model", True, tools) + + +@pytest.mark.parametrize("tools", [None, [], [_FUNCTION_TOOL, {"type": "custom", "custom": {"name": "patch"}}]]) +def test_drop_params_drops_the_choice_when_the_caller_runs_every_tool(tools: list[object] | None): + assert drops_unsupported_allowed_tools_tool_choice(_FLAT_CHOICE, "anthropic", "the-model", True, tools) diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index e70cb410dff..fa7438885cb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -591,6 +591,56 @@ async def test_responses_reports_auto_for_an_allowed_tools_choice_it_dropped( assert response.tool_choice == "auto" +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.asyncio +async def test_responses_keep_the_400_when_the_provider_would_run_an_mcp_tool(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"): + await litellm.aresponses( + model="anthropic/claude-haiku-4-5", + input="What is the weather?", + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}], + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + api_key="sk-ant-fake", + ) + + assert not respx_mock.calls + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_responses_report_auto_when_an_old_azure_api_version_dropped_the_choice(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(url__regex=r"https://azure-bridge\.example\.com/.*chat/completions.*").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-old-azure", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + response: Final = litellm.responses( + model="azure/gpt-4o", + input="What is the weather?", + tools=list(_RESPONSES_TOOLS), + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + use_chat_completions_api=True, + api_key="fake", + api_base="https://azure-bridge.example.com", + api_version="2023-07-01-preview", + ) + + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert response.tool_choice == "auto" + + def test_streamed_responses_report_auto_for_an_allowed_tools_choice_it_dropped(respx_mock: respx.MockRouter) -> None: respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( return_value=httpx.Response( diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index 0cc5bf20773..b48be8c57cb 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -1517,3 +1517,93 @@ async def test_mcp_tools_keep_the_400_for_an_allowed_tools_choice_the_provider_c execute.assert_not_awaited() assert not respx_mock.calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_mcp_tools_keep_the_400_on_an_azure_api_version_that_drops_the_choice( + respx_mock: respx.MockRouter, stream: bool +): + execute: Final = AsyncMock(return_value=[]) + + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([_MCP_ECHO_TOOL], {"echo": "demo"})), + ), + patch.object(LiteLLM_Proxy_MCP_Handler, "execute_tool_calls", execute), + pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"), + ): + await litellm.acompletion( + model="azure/gpt-4o", + messages=[{"role": "user", "content": "Call echo"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice=_FLAT_ALLOWED_ECHO, + drop_params=True, + stream=stream, + api_key="fake", + api_base="https://mcp-chat-rounds.example.com", + api_version="2023-07-01-preview", + ) + + execute.assert_not_awaited() + assert not respx_mock.calls + + +_ALLOWED_ONLY_ANOTHER_TOOL: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "another_tool"}], +} + + +def _served_by_a_recording_mcp_server(monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock) -> None: + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) + ) + monkeypatch.setattr( + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", + SimpleNamespace( + catalog=SimpleNamespace(operation=nullcontext), + get_registry=MagicMock(return_value={}), + call_tool=call_tool, + get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_an_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE) if stream else _ok(_CHAT_ROUND_ONE),)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + async def call_with_the_choice() -> None: + response: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice=_ALLOWED_ONLY_ANOTHER_TOOL, + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + stream=stream, + ) + if stream: + await _read_all(response) + + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([_MCP_ECHO_TOOL], {"echo": "demo"})), + ), + pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"), + ): + await call_with_the_choice() + + call_tool.assert_not_awaited() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 29aed3be8a8..bfe6d749828 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -533,6 +533,22 @@ async def test_a_malformed_allowed_tools_tool_choice_stays_a_400_on_mcp_gateway_ assert not respx_mock.calls +@pytest.mark.asyncio +async def test_drop_params_keeps_the_400_when_the_provider_would_run_an_mcp_tool(respx_mock: respx.MockRouter): + with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it") as exc: + await litellm.acompletion( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "What time is it?"}], + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + drop_params=True, + api_key="sk-ant-fake", + ) + + assert exc.value.status_code == 400 + assert not respx_mock.calls + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", diff --git a/tests/unit/test_tool_loop.py b/tests/unit/test_tool_loop.py index 916482e4b14..f4a6323ab9a 100644 --- a/tests/unit/test_tool_loop.py +++ b/tests/unit/test_tool_loop.py @@ -11,6 +11,7 @@ from litellm.types.llms.openai import ChatCompletionToolMessage from litellm.types.utils import ChatCompletionMessageToolCall OPENAI_CHAT_COMPLETIONS_URL: Final = "https://api.openai.com/v1/chat/completions" +ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages" WEATHER_TOOLS: Final = ( { "type": "function", @@ -248,57 +249,147 @@ def test_stream_rejected_before_any_request(respx_mock: respx.MockRouter) -> Non assert route.call_count == 0 -_FLAT_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "a"}]} +_ALLOWED_WEATHER_ONLY: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_weather"}], +} _EMPTY_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": []} +_TIME_CALL: Final = _tool_call("call_1", "get_time", {}) -@pytest.mark.parametrize( - ("model", "tool_choice"), - [("anthropic/claude-haiku-4-5", _FLAT_ALLOWED_TOOLS), ("openai/gpt-5-mini", _EMPTY_ALLOWED_TOOLS)], -) -def test_allowed_tools_choice_nothing_enforces_is_refused_under_drop_params( - respx_mock: respx.MockRouter, model: str, tool_choice: dict[str, object] -) -> None: +def _anthropic_message(*content: dict[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_tool_loop", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": list(content), + "stop_reason": "tool_use" if content[0]["type"] == "tool_use" else "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +def _anthropic_tool_use(name: str) -> httpx.Response: + return _anthropic_message({"type": "tool_use", "id": "toolu_1", "name": name, "input": {}}) + + +def _tools_run_before_the_refusal( + tool_choice: dict[str, object], **completion_kwargs: object +) -> list[ChatCompletionMessageToolCall]: executed: Final[list[ChatCompletionMessageToolCall]] = [] def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: executed.append(tool_call) return _tool_result(tool_call) - with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"): + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): litellm.run_tool_loop( - model=model, messages=[{"role": "user", "content": "hi"}], tools=WEATHER_TOOLS, execute_tool=execute, tool_choice=tool_choice, + api_key="sk-test", + **completion_kwargs, + ) + return executed + + +def test_a_tool_the_dropped_choice_excludes_never_runs(respx_mock: respx.MockRouter) -> None: + respx_mock.post(ANTHROPIC_MESSAGES_URL).mock(return_value=_anthropic_tool_use("get_time")) + + assert not _tools_run_before_the_refusal( + _ALLOWED_WEATHER_ONLY, model="anthropic/claude-haiku-4-5", drop_params=True + ) + assert "tool_choice" not in _request_bodies(respx_mock)[0] + + +def test_a_tool_the_dropped_choice_lists_still_runs(respx_mock: respx.MockRouter) -> None: + respx_mock.post(ANTHROPIC_MESSAGES_URL).mock( + side_effect=[_anthropic_tool_use("get_weather"), _anthropic_message({"type": "text", "text": "done"})] + ) + + assert ( + litellm.run_tool_loop( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=_ALLOWED_WEATHER_ONLY, + drop_params=True, + api_key="sk-test", + ) + == "done" + ) + + +@pytest.mark.parametrize( + ("tool_choice", "completion_kwargs"), + [(_ALLOWED_WEATHER_ONLY, {}), (_EMPTY_ALLOWED_TOOLS, {"drop_params": True})], +) +def test_a_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, tool_choice: dict[str, object], completion_kwargs: dict[str, object] +) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [_TIME_CALL])) + ) + + assert not _tools_run_before_the_refusal(tool_choice, model="openai/gpt-5-mini", **completion_kwargs) + + +def test_a_choice_on_a_model_alias_reaches_the_loop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "model_alias_map", {"the-alias": "openai/gpt-5-mini"}) + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response("done")) + ) + + assert ( + litellm.run_tool_loop( + model="the-alias", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=_ALLOWED_WEATHER_ONLY, + api_key="sk-test", + ) + == "done" + ) + + +@pytest.mark.asyncio +async def test_async_loop_never_runs_a_tool_the_choice_excludes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler()) + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [_TIME_CALL])) + ) + executed: Final[list[ChatCompletionMessageToolCall]] = [] + + async def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tool_call) + return _tool_result(tool_call) + + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): + await litellm.arun_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=execute, + tool_choice=_ALLOWED_WEATHER_ONLY, drop_params=True, api_key="sk-test", ) assert not executed - assert not respx_mock.calls - - -@pytest.mark.asyncio -async def test_async_loop_refuses_an_allowed_tools_choice_the_provider_cannot_enforce( - respx_mock: respx.MockRouter, -) -> None: - async def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: - return _tool_result(tool_call) - - with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"): - await litellm.arun_tool_loop( - model="anthropic/claude-haiku-4-5", - messages=[{"role": "user", "content": "hi"}], - tools=WEATHER_TOOLS, - execute_tool=execute, - tool_choice=_FLAT_ALLOWED_TOOLS, - drop_params=True, - api_key="sk-test", - ) - - assert not respx_mock.calls def test_allowed_tools_choice_the_provider_enforces_reaches_it_every_round(respx_mock: respx.MockRouter) -> None: