fix(allowed_tools): tolerate a string additional_drop_params, keep the choice to OpenAI models on Azure AI, prune dropped tools from it, and cap the semantic filter

This commit is contained in:
mateo-berri 2026-10-28 13:57:38 -07:00
parent 8fed8cdd37
commit 2660d736d2
14 changed files with 122 additions and 53 deletions

View file

@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None
####### COMPLETION MODELS ###################
from typing import Set
open_ai_chat_completion_models: Set = set()
open_ai_chat_completion_models: Set[str] = set()
open_ai_text_completion_models: Set = set()
cohere_models: Set = set()
cohere_chat_models: Set = set()

View file

@ -1,4 +1,4 @@
from collections.abc import Mapping, Sequence, Set
from collections.abc import Iterable, Mapping, Sequence, Set
from typing import Final, Literal, TypeVar
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
@ -127,9 +127,9 @@ def is_allowed_tools_choice(tool_choice: object) -> bool:
def _accepts_allowed_tools(custom_llm_provider: str, model: str) -> bool:
match custom_llm_provider:
case "azure_ai":
from litellm.llms.azure_ai.common_utils import azure_ai_routes_to_openai_chat
from litellm.llms.azure_ai.common_utils import azure_ai_serves_openai_chat_model
return azure_ai_routes_to_openai_chat(model)
return azure_ai_serves_openai_chat_model(model)
case _:
return custom_llm_provider in PROVIDERS_ACCEPTING_ALLOWED_TOOLS
@ -207,3 +207,8 @@ def pruned_allowed_tool_choice(request: object, removed_tool_names: Set[str]) ->
if len(kept) == len(tools):
return None
return _with_allowed_tools(choice, kept) if kept else "none"
def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]:
kept_names: Final = frozenset(filter(None, map(function_or_custom_tool_name, kept_tools)))
return frozenset(filter(None, map(function_or_custom_tool_name, request_tools))) - kept_names

View file

@ -1,4 +1,4 @@
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
@ -102,6 +102,15 @@ def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
def _with_kept_tools(
fields: Mapping[str, object], tools: Sequence[object], kept: Sequence[object]
) -> Mapping[str, object]:
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names
pruned: Final = pruned_allowed_tool_choice(fields, removed_tool_names(tools, kept))
return {**fields, "tools": kept} if pruned is None else {**fields, "tools": kept, "tool_choice": pruned}
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
tools: Final = data.get("tools")
if tools is None:
@ -116,7 +125,9 @@ def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
return None
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
return MalformedField(
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
"tools",
first_error,
lambda fields: _with_kept_tools(fields, entries, kept) if kept else _without_tool_fields(fields),
)

View file

@ -151,8 +151,10 @@ def azure_ai_supports_native_responses(model: str | None, api_base: str | None)
return AzureFoundryModelInfo.get_azure_ai_route(model) == "default"
def azure_ai_routes_to_openai_chat(model: str) -> bool:
return "claude" not in model.lower() and AzureFoundryModelInfo.get_azure_ai_route(model) != "agents"
def azure_ai_serves_openai_chat_model(model: str) -> bool:
return (
AzureFoundryModelInfo.get_azure_ai_route(model) == "default" and model in litellm.open_ai_chat_completion_models
)
def foundry_chat_rejects_function_tools_while_reasoning(

View file

@ -1,12 +1,11 @@
from __future__ import annotations
import json
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.litellm_core_utils.allowed_tools_tool_choice import function_or_custom_tool_name
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@ -295,11 +294,6 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]:
kept_names: Final = frozenset(filter(None, map(function_or_custom_tool_name, kept_tools)))
return frozenset(filter(None, map(function_or_custom_tool_name, request_tools))) - kept_names
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))

View file

@ -29,7 +29,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.completion_extras.litellm_responses_transformation.transformation import (
replayed_tool_search_tool_names,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -45,7 +45,6 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
openai_tool_name,
removed_tool_names,
role_out_of_guardrail_scope,
scoped_structured_message_indices,
stream_item_field,

View file

@ -47,7 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
OpenAiResponsesToChatCompletionStreamIterator,
tool_call_dict_from_output_item,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -59,7 +59,6 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
removed_tool_names,
role_out_of_guardrail_scope,
scoped_structured_message_indices,
stream_item_field,

View file

@ -114,6 +114,7 @@ from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
)
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.litellm_core_utils.tool_fields import drop_policy
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
from litellm.llms.base_llm.base_model_iterator import (
@ -5713,11 +5714,7 @@ def completion(
drops_unsupported_params: Final = (
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
)
drops_tool_choice: Final = (
drops_unsupported_params
or "tool_choice"
in _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(routing_kwargs.get("additional_drop_params") or [])
)
drops_tool_choice: Final = drop_policy(routing_kwargs).drops("tool_choice")
reasoning_model_group: Final = model_group_from_metadata(
routing_kwargs.get("metadata"), routing_kwargs.get("litellm_metadata")
)

View file

@ -179,13 +179,14 @@ class SemanticToolFilterHook(CustomLogger):
def _with_caller_allowed_tools(
self, request: object, available_tools: Sequence[object], filtered_tools: Sequence[object]
) -> tuple[object, ...]:
dropped_allowed_names: Final = allowed_tool_names(request) - frozenset(
self._selected_tool_names(filtered_tools)
allowed_names: Final = allowed_tool_names(request)
allowed_tools: Final = tuple(
tool for tool in available_tools if self.filter.extract_tool_info(tool)[0] in allowed_names
)
return (
*filtered_tools,
*(tool for tool in available_tools if self.filter.extract_tool_info(tool)[0] in dropped_allowed_names),
other_picks: Final = tuple(
tool for tool in filtered_tools if self.filter.extract_tool_info(tool)[0] not in allowed_names
)
return (*allowed_tools, *other_picks[: max(len(filtered_tools) - len(allowed_tools), 0)])
@staticmethod
async def _narrow_mcp_references(

View file

@ -8,6 +8,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
chat_allowed_tool_choice,
parse_allowed_tool_choice,
pruned_allowed_tool_choice,
removed_tool_names,
responses_allowed_tool_choice,
tool_choice_for_provider,
)
@ -209,3 +210,20 @@ def test_a_choice_left_with_no_tools_becomes_none():
)
def test_the_request_keeps_its_choice_when_nothing_it_allows_is_removed(request_body, removed):
assert pruned_allowed_tool_choice(request_body, removed) is None
@pytest.mark.parametrize(
"dropped_tool",
[
{"type": "function", "function": {"name": "get_time"}},
{"type": "custom", "custom": {"name": "get_time"}},
{"type": "function", "name": "get_time"},
{"type": "custom", "name": "get_time"},
],
ids=["chat_function", "chat_custom", "responses_function", "responses_custom"],
)
def test_a_dropped_function_or_custom_tool_is_named_as_removed(dropped_tool):
kept_tool: Final = {"type": "function", "function": {"name": "get_weather"}}
request_tools: Final = (kept_tool, dropped_tool, {"type": "mcp", "server_label": "deepwiki"})
assert removed_tool_names(request_tools, (kept_tool,)) == {"get_time"}

View file

@ -146,6 +146,51 @@ async def test_additional_drop_params_drops_only_the_malformed_tool_entries() ->
assert sent["tools"] == [_GOOD_TOOL]
@pytest.mark.parametrize(
("tool_choice", "pruned"),
[
(
{
"type": "allowed_tools",
"allowed_tools": {
"mode": "auto",
"tools": [
{"type": "function", "function": {"name": "get_weather"}},
{"type": "function", "function": {"name": "get_time"}},
],
},
},
{
"type": "allowed_tools",
"allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "get_weather"}}]},
},
),
(
{
"type": "allowed_tools",
"mode": "auto",
"tools": [{"type": "function", "name": "get_weather"}, {"type": "function", "name": "get_time"}],
},
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]},
),
({"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, "none"),
],
ids=["chat_shape", "responses_shape", "only_the_dropped_tool"],
)
@pytest.mark.asyncio
async def test_drop_params_prunes_a_dropped_malformed_tool_from_the_allowed_tools_choice(
tool_choice: Mapping[str, object], pruned: object
) -> None:
malformed_time_tool: Final = {"type": "function", "function": {"name": "get_time", "parameters": "object"}}
request: Final = _proxy_request(
"drops", drop_params=True, tools=[_GOOD_TOOL, malformed_time_tool], tool_choice=tool_choice
)
sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion")
assert (sent["tools"], sent["tool_choice"]) == ([_GOOD_TOOL], pruned)
@pytest.mark.asyncio
async def test_an_sdk_call_keeps_the_tools_it_passed() -> None:
tools: Final = (_GOOD_TOOL,)

View file

@ -6,7 +6,6 @@ import litellm
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
removed_tool_names,
)
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import LakeraAIGuardrail
@ -33,20 +32,3 @@ def test_a_per_guardrail_skip_flag_wins_over_the_global_setting(
assert effective_skip_system_message_for_guardrail(guardrail) is expected
assert effective_skip_tool_message_for_guardrail(guardrail) is expected
@pytest.mark.parametrize(
"dropped_tool",
[
{"type": "function", "function": {"name": "get_time"}},
{"type": "custom", "custom": {"name": "get_time"}},
{"type": "function", "name": "get_time"},
{"type": "custom", "name": "get_time"},
],
ids=["chat_function", "chat_custom", "responses_function", "responses_custom"],
)
def test_a_dropped_function_or_custom_tool_is_named_as_removed(dropped_tool):
kept_tool: Final = {"type": "function", "function": {"name": "get_weather"}}
request_tools: Final = (kept_tool, dropped_tool, {"type": "mcp", "server_label": "deepwiki"})
assert removed_tool_names(request_tools, (kept_tool,)) == {"get_time"}

View file

@ -629,7 +629,7 @@ async def test_semantic_filter_hook_keeps_mcp_tools_the_allowed_tools_choice_nam
MCPTool(name=f"mcp_tool_{i}", description=f"MCP tool {i}", inputSchema={"type": "object"}) for i in range(5)
]
filter_instance._build_router(mcp_tools)
picked = {tool.name for tool in await filter_instance.filter_tools(query="What is the weather?", available_tools=mcp_tools)}
picked = [tool.name for tool in await filter_instance.filter_tools(query="What is the weather?", available_tools=mcp_tools)]
unpicked = next(tool.name for tool in mcp_tools if tool.name not in picked)
native_tool = {"type": "function", "function": {"name": "get_current_weather", "parameters": {"type": "object"}}}
tool_choice = {
@ -651,7 +651,7 @@ async def test_semantic_filter_hook_keeps_mcp_tools_the_allowed_tools_choice_nam
call_type="completion",
)
assert {tool.name for tool in result["tools"] if isinstance(tool, MCPTool)} == {*picked, unpicked}
assert [tool.name for tool in result["tools"] if isinstance(tool, MCPTool)] == [unpicked, *picked[:-1]]
assert native_tool in result["tools"]
assert result["tool_choice"] == tool_choice
@ -674,7 +674,7 @@ async def test_semantic_filter_hook_keeps_expanded_tools_the_allowed_tools_choic
}
for i in range(5)
]
picked = {tool["name"] for tool in await filter_instance.filter_tools(query="Send an email", available_tools=expanded_tools)}
picked = [tool["name"] for tool in await filter_instance.filter_tools(query="Send an email", available_tools=expanded_tools)]
unpicked = next(tool["name"] for tool in expanded_tools if tool["name"] not in picked)
tool_choice = {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": unpicked}]}
@ -692,9 +692,9 @@ async def test_semantic_filter_hook_keeps_expanded_tools_the_allowed_tools_choic
call_type="aresponses",
)
assert set(result["tools"][0]["allowed_tools"]) == {*picked, unpicked}
assert result["tools"][0]["allowed_tools"] == [unpicked, *picked[:-1]]
assert result["tool_choice"] == tool_choice
assert result["metadata"]["litellm_semantic_filter_stats"] == f"5->{len(picked) + 1}"
assert result["metadata"]["litellm_semantic_filter_stats"] == f"5->{len(picked)}"
@requires_semantic_router

View file

@ -5540,7 +5540,7 @@ def _mock_chat_completion(respx_mock: respx.MockRouter) -> respx.Route:
[
("openai/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
("litellm_proxy/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
("azure_ai/DeepSeek-V4-Flash", {"api_base": "http://localhost:12346/models"}),
("azure_ai/gpt-4.1-mini", {"api_base": "http://localhost:12346/models"}),
("azure/gpt-4.1-mini", {"api_base": "http://localhost:12346", "api_version": "2025-04-01-preview"}),
],
)
@ -5570,6 +5570,7 @@ def test_completion_sends_allowed_tools_nested_to_openai_shaped_providers(
"ollama/llama3",
"deepseek/deepseek-chat",
"azure_ai/claude-haiku-4-5",
"azure_ai/DeepSeek-V4-Flash",
"azure_ai/agents/asst_abc123",
],
)
@ -5617,6 +5618,21 @@ def test_drop_params_sends_tools_without_an_allowed_tools_choice_a_provider_cann
assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_weather", "get_time"]
def test_completion_sends_tools_when_additional_drop_params_is_a_string(respx_mock: respx.MockRouter) -> None:
mock_route: Final = _mock_chat_completion(respx_mock)
litellm.completion(
model="openai/gpt-4.1-mini",
messages=[{"role": "user", "content": "What time is it?"}],
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
api_base="http://localhost:12346/v1",
api_key="fake_api_key",
additional_drop_params="logprobs",
)
assert mock_route.call_count == 1
@pytest.mark.parametrize("model", ["openai/gpt-4.1-mini", "deepseek/deepseek-chat"])
def test_drop_params_sends_tools_without_a_malformed_allowed_tools_choice(respx_mock: respx.MockRouter, model: str) -> None:
mock_route: Final = _mock_chat_completion(respx_mock)