From 2660d736d249baf3cb8b290be2e8650e97601643 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:57:38 -0700 Subject: [PATCH] fix(allowed_tools): tolerate a string additional_drop_params, keep the choice to OpenAI models on Azure AI, prune dropped tools from it, and cap the semantic filter --- litellm/__init__.py | 2 +- .../allowed_tools_tool_choice.py | 11 +++-- litellm/litellm_core_utils/tool_fields.py | 15 ++++++- litellm/llms/azure_ai/common_utils.py | 6 ++- .../base_llm/guardrail_translation/utils.py | 8 +--- .../chat/guardrail_translation/handler.py | 3 +- .../guardrail_translation/handler.py | 3 +- litellm/main.py | 7 +-- .../proxy/hooks/mcp_semantic_filter/hook.py | 11 ++--- .../test_allowed_tools_tool_choice.py | 18 ++++++++ .../litellm_core_utils/test_tool_fields.py | 45 +++++++++++++++++++ .../guardrail_translation/test_utils.py | 18 -------- .../mcp_server/test_semantic_tool_filter.py | 10 ++--- tests/unit/test_main.py | 18 +++++++- 14 files changed, 122 insertions(+), 53 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 41e3fa34323..125d3401855 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set -open_ai_chat_completion_models: Set = set() +open_ai_chat_completion_models: Set[str] = set() open_ai_text_completion_models: Set = set() cohere_models: Set = set() cohere_chat_models: Set = set() diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 510cd4a6cb6..860d0932112 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence, Set +from collections.abc import Iterable, Mapping, Sequence, Set from typing import Final, Literal, TypeVar from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam @@ -127,9 +127,9 @@ def is_allowed_tools_choice(tool_choice: object) -> bool: def _accepts_allowed_tools(custom_llm_provider: str, model: str) -> bool: match custom_llm_provider: case "azure_ai": - from litellm.llms.azure_ai.common_utils import azure_ai_routes_to_openai_chat + from litellm.llms.azure_ai.common_utils import azure_ai_serves_openai_chat_model - return azure_ai_routes_to_openai_chat(model) + return azure_ai_serves_openai_chat_model(model) case _: return custom_llm_provider in PROVIDERS_ACCEPTING_ALLOWED_TOOLS @@ -207,3 +207,8 @@ def pruned_allowed_tool_choice(request: object, removed_tool_names: Set[str]) -> if len(kept) == len(tools): return None return _with_allowed_tools(choice, kept) if kept else "none" + + +def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]: + kept_names: Final = frozenset(filter(None, map(function_or_custom_tool_name, kept_tools))) + return frozenset(filter(None, map(function_or_custom_tool_name, request_tools))) - kept_names diff --git a/litellm/litellm_core_utils/tool_fields.py b/litellm/litellm_core_utils/tool_fields.py index f90c8f5d174..4151b85d092 100644 --- a/litellm/litellm_core_utils/tool_fields.py +++ b/litellm/litellm_core_utils/tool_fields.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from functools import reduce from typing import Annotated, Final, TypeAlias @@ -102,6 +102,15 @@ def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]: return {name: value for name, value in data.items() if name not in TOOL_FIELDS} +def _with_kept_tools( + fields: Mapping[str, object], tools: Sequence[object], kept: Sequence[object] +) -> Mapping[str, object]: + from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names + + pruned: Final = pruned_allowed_tool_choice(fields, removed_tool_names(tools, kept)) + return {**fields, "tools": kept} if pruned is None else {**fields, "tools": kept, "tool_choice": pruned} + + def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None: tools: Final = data.get("tools") if tools is None: @@ -116,7 +125,9 @@ def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None: return None kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None] return MalformedField( - "tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields) + "tools", + first_error, + lambda fields: _with_kept_tools(fields, entries, kept) if kept else _without_tool_fields(fields), ) diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index b4bf8163721..7722059aa2c 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -151,8 +151,10 @@ def azure_ai_supports_native_responses(model: str | None, api_base: str | None) return AzureFoundryModelInfo.get_azure_ai_route(model) == "default" -def azure_ai_routes_to_openai_chat(model: str) -> bool: - return "claude" not in model.lower() and AzureFoundryModelInfo.get_azure_ai_route(model) != "agents" +def azure_ai_serves_openai_chat_model(model: str) -> bool: + return ( + AzureFoundryModelInfo.get_azure_ai_route(model) == "default" and model in litellm.open_ai_chat_completion_models + ) def foundry_chat_rejects_function_tools_while_reasoning( diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index ad86901e5fe..725146fe894 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -1,12 +1,11 @@ from __future__ import annotations import json -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError -from litellm.litellm_core_utils.allowed_tools_tool_choice import function_or_custom_tool_name from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage @@ -295,11 +294,6 @@ def openai_tool_name(tool: object) -> str | None: return flat_name if isinstance(flat_name, str) else None -def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]: - kept_names: Final = frozenset(filter(None, map(function_or_custom_tool_name, kept_tools))) - return frozenset(filter(None, map(function_or_custom_tool_name, request_tools))) - kept_names - - _FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True)) _ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True)) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 455f695364f..d699426c785 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -29,7 +29,7 @@ from litellm._logging import verbose_proxy_logger from litellm.completion_extras.litellm_responses_transformation.transformation import ( replayed_tool_search_tool_names, ) -from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice +from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, @@ -45,7 +45,6 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, openai_tool_name, - removed_tool_names, role_out_of_guardrail_scope, scoped_structured_message_indices, stream_item_field, diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 8393e1772c6..cc0a3b9e403 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -47,7 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i OpenAiResponsesToChatCompletionStreamIterator, tool_call_dict_from_output_item, ) -from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice +from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, @@ -59,7 +59,6 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, list_field, merge_guardrailed_scoped_messages, - removed_tool_names, role_out_of_guardrail_scope, scoped_structured_message_indices, stream_item_field, diff --git a/litellm/main.py b/litellm/main.py index 1c751de5107..e282cec5700 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -114,6 +114,7 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +from litellm.litellm_core_utils.tool_fields import drop_policy from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( @@ -5713,11 +5714,7 @@ def completion( drops_unsupported_params: Final = ( litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True ) - drops_tool_choice: Final = ( - drops_unsupported_params - or "tool_choice" - in _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(routing_kwargs.get("additional_drop_params") or []) - ) + drops_tool_choice: Final = drop_policy(routing_kwargs).drops("tool_choice") reasoning_model_group: Final = model_group_from_metadata( routing_kwargs.get("metadata"), routing_kwargs.get("litellm_metadata") ) diff --git a/litellm/proxy/hooks/mcp_semantic_filter/hook.py b/litellm/proxy/hooks/mcp_semantic_filter/hook.py index 1cd8f2e744a..8ac52281850 100644 --- a/litellm/proxy/hooks/mcp_semantic_filter/hook.py +++ b/litellm/proxy/hooks/mcp_semantic_filter/hook.py @@ -179,13 +179,14 @@ class SemanticToolFilterHook(CustomLogger): def _with_caller_allowed_tools( self, request: object, available_tools: Sequence[object], filtered_tools: Sequence[object] ) -> tuple[object, ...]: - dropped_allowed_names: Final = allowed_tool_names(request) - frozenset( - self._selected_tool_names(filtered_tools) + allowed_names: Final = allowed_tool_names(request) + allowed_tools: Final = tuple( + tool for tool in available_tools if self.filter.extract_tool_info(tool)[0] in allowed_names ) - return ( - *filtered_tools, - *(tool for tool in available_tools if self.filter.extract_tool_info(tool)[0] in dropped_allowed_names), + other_picks: Final = tuple( + tool for tool in filtered_tools if self.filter.extract_tool_info(tool)[0] not in allowed_names ) + return (*allowed_tools, *other_picks[: max(len(filtered_tools) - len(allowed_tools), 0)]) @staticmethod async def _narrow_mcp_references( diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index fa3015447f2..f22926c3026 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( chat_allowed_tool_choice, parse_allowed_tool_choice, pruned_allowed_tool_choice, + removed_tool_names, responses_allowed_tool_choice, tool_choice_for_provider, ) @@ -209,3 +210,20 @@ def test_a_choice_left_with_no_tools_becomes_none(): ) def test_the_request_keeps_its_choice_when_nothing_it_allows_is_removed(request_body, removed): assert pruned_allowed_tool_choice(request_body, removed) is None + + +@pytest.mark.parametrize( + "dropped_tool", + [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "custom": {"name": "get_time"}}, + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "get_time"}, + ], + ids=["chat_function", "chat_custom", "responses_function", "responses_custom"], +) +def test_a_dropped_function_or_custom_tool_is_named_as_removed(dropped_tool): + kept_tool: Final = {"type": "function", "function": {"name": "get_weather"}} + request_tools: Final = (kept_tool, dropped_tool, {"type": "mcp", "server_label": "deepwiki"}) + + assert removed_tool_names(request_tools, (kept_tool,)) == {"get_time"} diff --git a/tests/unit/litellm_core_utils/test_tool_fields.py b/tests/unit/litellm_core_utils/test_tool_fields.py index 23d7de98fbc..b27b9f10b7e 100644 --- a/tests/unit/litellm_core_utils/test_tool_fields.py +++ b/tests/unit/litellm_core_utils/test_tool_fields.py @@ -146,6 +146,51 @@ async def test_additional_drop_params_drops_only_the_malformed_tool_entries() -> assert sent["tools"] == [_GOOD_TOOL] +@pytest.mark.parametrize( + ("tool_choice", "pruned"), + [ + ( + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [ + {"type": "function", "function": {"name": "get_weather"}}, + {"type": "function", "function": {"name": "get_time"}}, + ], + }, + }, + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "get_weather"}}]}, + }, + ), + ( + { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_weather"}, {"type": "function", "name": "get_time"}], + }, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]}, + ), + ({"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, "none"), + ], + ids=["chat_shape", "responses_shape", "only_the_dropped_tool"], +) +@pytest.mark.asyncio +async def test_drop_params_prunes_a_dropped_malformed_tool_from_the_allowed_tools_choice( + tool_choice: Mapping[str, object], pruned: object +) -> None: + malformed_time_tool: Final = {"type": "function", "function": {"name": "get_time", "parameters": "object"}} + request: Final = _proxy_request( + "drops", drop_params=True, tools=[_GOOD_TOOL, malformed_time_tool], tool_choice=tool_choice + ) + + sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion") + + assert (sent["tools"], sent["tool_choice"]) == ([_GOOD_TOOL], pruned) + + @pytest.mark.asyncio async def test_an_sdk_call_keeps_the_tools_it_passed() -> None: tools: Final = (_GOOD_TOOL,) diff --git a/tests/unit/llms/base_llm/guardrail_translation/test_utils.py b/tests/unit/llms/base_llm/guardrail_translation/test_utils.py index a246d37819e..65208a2bd3b 100644 --- a/tests/unit/llms/base_llm/guardrail_translation/test_utils.py +++ b/tests/unit/llms/base_llm/guardrail_translation/test_utils.py @@ -6,7 +6,6 @@ import litellm from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, - removed_tool_names, ) from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import LakeraAIGuardrail @@ -33,20 +32,3 @@ def test_a_per_guardrail_skip_flag_wins_over_the_global_setting( assert effective_skip_system_message_for_guardrail(guardrail) is expected assert effective_skip_tool_message_for_guardrail(guardrail) is expected - - -@pytest.mark.parametrize( - "dropped_tool", - [ - {"type": "function", "function": {"name": "get_time"}}, - {"type": "custom", "custom": {"name": "get_time"}}, - {"type": "function", "name": "get_time"}, - {"type": "custom", "name": "get_time"}, - ], - ids=["chat_function", "chat_custom", "responses_function", "responses_custom"], -) -def test_a_dropped_function_or_custom_tool_is_named_as_removed(dropped_tool): - kept_tool: Final = {"type": "function", "function": {"name": "get_weather"}} - request_tools: Final = (kept_tool, dropped_tool, {"type": "mcp", "server_label": "deepwiki"}) - - assert removed_tool_names(request_tools, (kept_tool,)) == {"get_time"} diff --git a/tests/unit/proxy/mcp_server/test_semantic_tool_filter.py b/tests/unit/proxy/mcp_server/test_semantic_tool_filter.py index 40c47d52e2b..fd3c3673d68 100644 --- a/tests/unit/proxy/mcp_server/test_semantic_tool_filter.py +++ b/tests/unit/proxy/mcp_server/test_semantic_tool_filter.py @@ -629,7 +629,7 @@ async def test_semantic_filter_hook_keeps_mcp_tools_the_allowed_tools_choice_nam MCPTool(name=f"mcp_tool_{i}", description=f"MCP tool {i}", inputSchema={"type": "object"}) for i in range(5) ] filter_instance._build_router(mcp_tools) - picked = {tool.name for tool in await filter_instance.filter_tools(query="What is the weather?", available_tools=mcp_tools)} + picked = [tool.name for tool in await filter_instance.filter_tools(query="What is the weather?", available_tools=mcp_tools)] unpicked = next(tool.name for tool in mcp_tools if tool.name not in picked) native_tool = {"type": "function", "function": {"name": "get_current_weather", "parameters": {"type": "object"}}} tool_choice = { @@ -651,7 +651,7 @@ async def test_semantic_filter_hook_keeps_mcp_tools_the_allowed_tools_choice_nam call_type="completion", ) - assert {tool.name for tool in result["tools"] if isinstance(tool, MCPTool)} == {*picked, unpicked} + assert [tool.name for tool in result["tools"] if isinstance(tool, MCPTool)] == [unpicked, *picked[:-1]] assert native_tool in result["tools"] assert result["tool_choice"] == tool_choice @@ -674,7 +674,7 @@ async def test_semantic_filter_hook_keeps_expanded_tools_the_allowed_tools_choic } for i in range(5) ] - picked = {tool["name"] for tool in await filter_instance.filter_tools(query="Send an email", available_tools=expanded_tools)} + picked = [tool["name"] for tool in await filter_instance.filter_tools(query="Send an email", available_tools=expanded_tools)] unpicked = next(tool["name"] for tool in expanded_tools if tool["name"] not in picked) tool_choice = {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": unpicked}]} @@ -692,9 +692,9 @@ async def test_semantic_filter_hook_keeps_expanded_tools_the_allowed_tools_choic call_type="aresponses", ) - assert set(result["tools"][0]["allowed_tools"]) == {*picked, unpicked} + assert result["tools"][0]["allowed_tools"] == [unpicked, *picked[:-1]] assert result["tool_choice"] == tool_choice - assert result["metadata"]["litellm_semantic_filter_stats"] == f"5->{len(picked) + 1}" + assert result["metadata"]["litellm_semantic_filter_stats"] == f"5->{len(picked)}" @requires_semantic_router diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 0198b73a60b..69e119ba208 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5540,7 +5540,7 @@ def _mock_chat_completion(respx_mock: respx.MockRouter) -> respx.Route: [ ("openai/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}), ("litellm_proxy/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}), - ("azure_ai/DeepSeek-V4-Flash", {"api_base": "http://localhost:12346/models"}), + ("azure_ai/gpt-4.1-mini", {"api_base": "http://localhost:12346/models"}), ("azure/gpt-4.1-mini", {"api_base": "http://localhost:12346", "api_version": "2025-04-01-preview"}), ], ) @@ -5570,6 +5570,7 @@ def test_completion_sends_allowed_tools_nested_to_openai_shaped_providers( "ollama/llama3", "deepseek/deepseek-chat", "azure_ai/claude-haiku-4-5", + "azure_ai/DeepSeek-V4-Flash", "azure_ai/agents/asst_abc123", ], ) @@ -5617,6 +5618,21 @@ def test_drop_params_sends_tools_without_an_allowed_tools_choice_a_provider_cann assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_weather", "get_time"] +def test_completion_sends_tools_when_additional_drop_params_is_a_string(respx_mock: respx.MockRouter) -> None: + mock_route: Final = _mock_chat_completion(respx_mock) + + litellm.completion( + model="openai/gpt-4.1-mini", + messages=[{"role": "user", "content": "What time is it?"}], + tools=ALLOWED_TOOLS_REQUEST_TOOLS, + api_base="http://localhost:12346/v1", + api_key="fake_api_key", + additional_drop_params="logprobs", + ) + + assert mock_route.call_count == 1 + + @pytest.mark.parametrize("model", ["openai/gpt-4.1-mini", "deepseek/deepseek-chat"]) def test_drop_params_sends_tools_without_a_malformed_allowed_tools_choice(respx_mock: respx.MockRouter, model: str) -> None: mock_route: Final = _mock_chat_completion(respx_mock)