mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(allowed_tools): tolerate a string additional_drop_params, keep the choice to OpenAI models on Azure AI, prune dropped tools from it, and cap the semantic filter
This commit is contained in:
parent
8fed8cdd37
commit
2660d736d2
14 changed files with 122 additions and 53 deletions
|
|
@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None
|
|||
####### COMPLETION MODELS ###################
|
||||
from typing import Set
|
||||
|
||||
open_ai_chat_completion_models: Set = set()
|
||||
open_ai_chat_completion_models: Set[str] = set()
|
||||
open_ai_text_completion_models: Set = set()
|
||||
cohere_models: Set = set()
|
||||
cohere_chat_models: Set = set()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Mapping, Sequence, Set
|
||||
from collections.abc import Iterable, Mapping, Sequence, Set
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
|
||||
|
|
@ -127,9 +127,9 @@ def is_allowed_tools_choice(tool_choice: object) -> bool:
|
|||
def _accepts_allowed_tools(custom_llm_provider: str, model: str) -> bool:
|
||||
match custom_llm_provider:
|
||||
case "azure_ai":
|
||||
from litellm.llms.azure_ai.common_utils import azure_ai_routes_to_openai_chat
|
||||
from litellm.llms.azure_ai.common_utils import azure_ai_serves_openai_chat_model
|
||||
|
||||
return azure_ai_routes_to_openai_chat(model)
|
||||
return azure_ai_serves_openai_chat_model(model)
|
||||
case _:
|
||||
return custom_llm_provider in PROVIDERS_ACCEPTING_ALLOWED_TOOLS
|
||||
|
||||
|
|
@ -207,3 +207,8 @@ def pruned_allowed_tool_choice(request: object, removed_tool_names: Set[str]) ->
|
|||
if len(kept) == len(tools):
|
||||
return None
|
||||
return _with_allowed_tools(choice, kept) if kept else "none"
|
||||
|
||||
|
||||
def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]:
|
||||
kept_names: Final = frozenset(filter(None, map(function_or_custom_tool_name, kept_tools)))
|
||||
return frozenset(filter(None, map(function_or_custom_tool_name, request_tools))) - kept_names
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from typing import Annotated, Final, TypeAlias
|
||||
|
|
@ -102,6 +102,15 @@ def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
|
|||
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
|
||||
|
||||
|
||||
def _with_kept_tools(
|
||||
fields: Mapping[str, object], tools: Sequence[object], kept: Sequence[object]
|
||||
) -> Mapping[str, object]:
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names
|
||||
|
||||
pruned: Final = pruned_allowed_tool_choice(fields, removed_tool_names(tools, kept))
|
||||
return {**fields, "tools": kept} if pruned is None else {**fields, "tools": kept, "tool_choice": pruned}
|
||||
|
||||
|
||||
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
|
||||
tools: Final = data.get("tools")
|
||||
if tools is None:
|
||||
|
|
@ -116,7 +125,9 @@ def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
|
|||
return None
|
||||
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
|
||||
return MalformedField(
|
||||
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
|
||||
"tools",
|
||||
first_error,
|
||||
lambda fields: _with_kept_tools(fields, entries, kept) if kept else _without_tool_fields(fields),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -151,8 +151,10 @@ def azure_ai_supports_native_responses(model: str | None, api_base: str | None)
|
|||
return AzureFoundryModelInfo.get_azure_ai_route(model) == "default"
|
||||
|
||||
|
||||
def azure_ai_routes_to_openai_chat(model: str) -> bool:
|
||||
return "claude" not in model.lower() and AzureFoundryModelInfo.get_azure_ai_route(model) != "agents"
|
||||
def azure_ai_serves_openai_chat_model(model: str) -> bool:
|
||||
return (
|
||||
AzureFoundryModelInfo.get_azure_ai_route(model) == "default" and model in litellm.open_ai_chat_completion_models
|
||||
)
|
||||
|
||||
|
||||
def foundry_chat_rejects_function_tools_while_reasoning(
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import function_or_custom_tool_name
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
|
||||
|
||||
|
|
@ -295,11 +294,6 @@ def openai_tool_name(tool: object) -> str | None:
|
|||
return flat_name if isinstance(flat_name, str) else None
|
||||
|
||||
|
||||
def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]:
|
||||
kept_names: Final = frozenset(filter(None, map(function_or_custom_tool_name, kept_tools)))
|
||||
return frozenset(filter(None, map(function_or_custom_tool_name, request_tools))) - kept_names
|
||||
|
||||
|
||||
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
|
||||
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
replayed_tool_search_tool_names,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
|
|
@ -45,7 +45,6 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
merge_guardrailed_scoped_messages,
|
||||
merge_returned_tools_into_request_tools,
|
||||
openai_tool_name,
|
||||
removed_tool_names,
|
||||
role_out_of_guardrail_scope,
|
||||
scoped_structured_message_indices,
|
||||
stream_item_field,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
|
|||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
tool_call_dict_from_output_item,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice, removed_tool_names
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
|
|
@ -59,7 +59,6 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
effective_skip_system_message_for_guardrail,
|
||||
list_field,
|
||||
merge_guardrailed_scoped_messages,
|
||||
removed_tool_names,
|
||||
role_out_of_guardrail_scope,
|
||||
scoped_structured_message_indices,
|
||||
stream_item_field,
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ from litellm.litellm_core_utils.request_timeout_resolver import (
|
|||
get_configured_request_timeout,
|
||||
)
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.litellm_core_utils.tool_fields import drop_policy
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
|
||||
from litellm.llms.base_llm.base_model_iterator import (
|
||||
|
|
@ -5713,11 +5714,7 @@ def completion(
|
|||
drops_unsupported_params: Final = (
|
||||
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
|
||||
)
|
||||
drops_tool_choice: Final = (
|
||||
drops_unsupported_params
|
||||
or "tool_choice"
|
||||
in _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(routing_kwargs.get("additional_drop_params") or [])
|
||||
)
|
||||
drops_tool_choice: Final = drop_policy(routing_kwargs).drops("tool_choice")
|
||||
reasoning_model_group: Final = model_group_from_metadata(
|
||||
routing_kwargs.get("metadata"), routing_kwargs.get("litellm_metadata")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -179,13 +179,14 @@ class SemanticToolFilterHook(CustomLogger):
|
|||
def _with_caller_allowed_tools(
|
||||
self, request: object, available_tools: Sequence[object], filtered_tools: Sequence[object]
|
||||
) -> tuple[object, ...]:
|
||||
dropped_allowed_names: Final = allowed_tool_names(request) - frozenset(
|
||||
self._selected_tool_names(filtered_tools)
|
||||
allowed_names: Final = allowed_tool_names(request)
|
||||
allowed_tools: Final = tuple(
|
||||
tool for tool in available_tools if self.filter.extract_tool_info(tool)[0] in allowed_names
|
||||
)
|
||||
return (
|
||||
*filtered_tools,
|
||||
*(tool for tool in available_tools if self.filter.extract_tool_info(tool)[0] in dropped_allowed_names),
|
||||
other_picks: Final = tuple(
|
||||
tool for tool in filtered_tools if self.filter.extract_tool_info(tool)[0] not in allowed_names
|
||||
)
|
||||
return (*allowed_tools, *other_picks[: max(len(filtered_tools) - len(allowed_tools), 0)])
|
||||
|
||||
@staticmethod
|
||||
async def _narrow_mcp_references(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
|||
chat_allowed_tool_choice,
|
||||
parse_allowed_tool_choice,
|
||||
pruned_allowed_tool_choice,
|
||||
removed_tool_names,
|
||||
responses_allowed_tool_choice,
|
||||
tool_choice_for_provider,
|
||||
)
|
||||
|
|
@ -209,3 +210,20 @@ def test_a_choice_left_with_no_tools_becomes_none():
|
|||
)
|
||||
def test_the_request_keeps_its_choice_when_nothing_it_allows_is_removed(request_body, removed):
|
||||
assert pruned_allowed_tool_choice(request_body, removed) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dropped_tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "get_time"}},
|
||||
{"type": "custom", "custom": {"name": "get_time"}},
|
||||
{"type": "function", "name": "get_time"},
|
||||
{"type": "custom", "name": "get_time"},
|
||||
],
|
||||
ids=["chat_function", "chat_custom", "responses_function", "responses_custom"],
|
||||
)
|
||||
def test_a_dropped_function_or_custom_tool_is_named_as_removed(dropped_tool):
|
||||
kept_tool: Final = {"type": "function", "function": {"name": "get_weather"}}
|
||||
request_tools: Final = (kept_tool, dropped_tool, {"type": "mcp", "server_label": "deepwiki"})
|
||||
|
||||
assert removed_tool_names(request_tools, (kept_tool,)) == {"get_time"}
|
||||
|
|
|
|||
|
|
@ -146,6 +146,51 @@ async def test_additional_drop_params_drops_only_the_malformed_tool_entries() ->
|
|||
assert sent["tools"] == [_GOOD_TOOL]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "pruned"),
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "auto",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
{"type": "function", "function": {"name": "get_time"}},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "get_weather"}}]},
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"mode": "auto",
|
||||
"tools": [{"type": "function", "name": "get_weather"}, {"type": "function", "name": "get_time"}],
|
||||
},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
),
|
||||
({"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, "none"),
|
||||
],
|
||||
ids=["chat_shape", "responses_shape", "only_the_dropped_tool"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_drop_params_prunes_a_dropped_malformed_tool_from_the_allowed_tools_choice(
|
||||
tool_choice: Mapping[str, object], pruned: object
|
||||
) -> None:
|
||||
malformed_time_tool: Final = {"type": "function", "function": {"name": "get_time", "parameters": "object"}}
|
||||
request: Final = _proxy_request(
|
||||
"drops", drop_params=True, tools=[_GOOD_TOOL, malformed_time_tool], tool_choice=tool_choice
|
||||
)
|
||||
|
||||
sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion")
|
||||
|
||||
assert (sent["tools"], sent["tool_choice"]) == ([_GOOD_TOOL], pruned)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_sdk_call_keeps_the_tools_it_passed() -> None:
|
||||
tools: Final = (_GOOD_TOOL,)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import litellm
|
|||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
effective_skip_system_message_for_guardrail,
|
||||
effective_skip_tool_message_for_guardrail,
|
||||
removed_tool_names,
|
||||
)
|
||||
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import LakeraAIGuardrail
|
||||
|
||||
|
|
@ -33,20 +32,3 @@ def test_a_per_guardrail_skip_flag_wins_over_the_global_setting(
|
|||
|
||||
assert effective_skip_system_message_for_guardrail(guardrail) is expected
|
||||
assert effective_skip_tool_message_for_guardrail(guardrail) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dropped_tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "get_time"}},
|
||||
{"type": "custom", "custom": {"name": "get_time"}},
|
||||
{"type": "function", "name": "get_time"},
|
||||
{"type": "custom", "name": "get_time"},
|
||||
],
|
||||
ids=["chat_function", "chat_custom", "responses_function", "responses_custom"],
|
||||
)
|
||||
def test_a_dropped_function_or_custom_tool_is_named_as_removed(dropped_tool):
|
||||
kept_tool: Final = {"type": "function", "function": {"name": "get_weather"}}
|
||||
request_tools: Final = (kept_tool, dropped_tool, {"type": "mcp", "server_label": "deepwiki"})
|
||||
|
||||
assert removed_tool_names(request_tools, (kept_tool,)) == {"get_time"}
|
||||
|
|
|
|||
|
|
@ -629,7 +629,7 @@ async def test_semantic_filter_hook_keeps_mcp_tools_the_allowed_tools_choice_nam
|
|||
MCPTool(name=f"mcp_tool_{i}", description=f"MCP tool {i}", inputSchema={"type": "object"}) for i in range(5)
|
||||
]
|
||||
filter_instance._build_router(mcp_tools)
|
||||
picked = {tool.name for tool in await filter_instance.filter_tools(query="What is the weather?", available_tools=mcp_tools)}
|
||||
picked = [tool.name for tool in await filter_instance.filter_tools(query="What is the weather?", available_tools=mcp_tools)]
|
||||
unpicked = next(tool.name for tool in mcp_tools if tool.name not in picked)
|
||||
native_tool = {"type": "function", "function": {"name": "get_current_weather", "parameters": {"type": "object"}}}
|
||||
tool_choice = {
|
||||
|
|
@ -651,7 +651,7 @@ async def test_semantic_filter_hook_keeps_mcp_tools_the_allowed_tools_choice_nam
|
|||
call_type="completion",
|
||||
)
|
||||
|
||||
assert {tool.name for tool in result["tools"] if isinstance(tool, MCPTool)} == {*picked, unpicked}
|
||||
assert [tool.name for tool in result["tools"] if isinstance(tool, MCPTool)] == [unpicked, *picked[:-1]]
|
||||
assert native_tool in result["tools"]
|
||||
assert result["tool_choice"] == tool_choice
|
||||
|
||||
|
|
@ -674,7 +674,7 @@ async def test_semantic_filter_hook_keeps_expanded_tools_the_allowed_tools_choic
|
|||
}
|
||||
for i in range(5)
|
||||
]
|
||||
picked = {tool["name"] for tool in await filter_instance.filter_tools(query="Send an email", available_tools=expanded_tools)}
|
||||
picked = [tool["name"] for tool in await filter_instance.filter_tools(query="Send an email", available_tools=expanded_tools)]
|
||||
unpicked = next(tool["name"] for tool in expanded_tools if tool["name"] not in picked)
|
||||
tool_choice = {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": unpicked}]}
|
||||
|
||||
|
|
@ -692,9 +692,9 @@ async def test_semantic_filter_hook_keeps_expanded_tools_the_allowed_tools_choic
|
|||
call_type="aresponses",
|
||||
)
|
||||
|
||||
assert set(result["tools"][0]["allowed_tools"]) == {*picked, unpicked}
|
||||
assert result["tools"][0]["allowed_tools"] == [unpicked, *picked[:-1]]
|
||||
assert result["tool_choice"] == tool_choice
|
||||
assert result["metadata"]["litellm_semantic_filter_stats"] == f"5->{len(picked) + 1}"
|
||||
assert result["metadata"]["litellm_semantic_filter_stats"] == f"5->{len(picked)}"
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
|
|
|
|||
|
|
@ -5540,7 +5540,7 @@ def _mock_chat_completion(respx_mock: respx.MockRouter) -> respx.Route:
|
|||
[
|
||||
("openai/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
|
||||
("litellm_proxy/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
|
||||
("azure_ai/DeepSeek-V4-Flash", {"api_base": "http://localhost:12346/models"}),
|
||||
("azure_ai/gpt-4.1-mini", {"api_base": "http://localhost:12346/models"}),
|
||||
("azure/gpt-4.1-mini", {"api_base": "http://localhost:12346", "api_version": "2025-04-01-preview"}),
|
||||
],
|
||||
)
|
||||
|
|
@ -5570,6 +5570,7 @@ def test_completion_sends_allowed_tools_nested_to_openai_shaped_providers(
|
|||
"ollama/llama3",
|
||||
"deepseek/deepseek-chat",
|
||||
"azure_ai/claude-haiku-4-5",
|
||||
"azure_ai/DeepSeek-V4-Flash",
|
||||
"azure_ai/agents/asst_abc123",
|
||||
],
|
||||
)
|
||||
|
|
@ -5617,6 +5618,21 @@ def test_drop_params_sends_tools_without_an_allowed_tools_choice_a_provider_cann
|
|||
assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_weather", "get_time"]
|
||||
|
||||
|
||||
def test_completion_sends_tools_when_additional_drop_params_is_a_string(respx_mock: respx.MockRouter) -> None:
|
||||
mock_route: Final = _mock_chat_completion(respx_mock)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/gpt-4.1-mini",
|
||||
messages=[{"role": "user", "content": "What time is it?"}],
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
api_base="http://localhost:12346/v1",
|
||||
api_key="fake_api_key",
|
||||
additional_drop_params="logprobs",
|
||||
)
|
||||
|
||||
assert mock_route.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["openai/gpt-4.1-mini", "deepseek/deepseek-chat"])
|
||||
def test_drop_params_sends_tools_without_a_malformed_allowed_tools_choice(respx_mock: respx.MockRouter, model: str) -> None:
|
||||
mock_route: Final = _mock_chat_completion(respx_mock)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue