fix(tool_choice): accept both allowed_tools shapes and gate unsupported providers

Validate the chat and Responses allowed_tools shapes with pydantic models, send the
nested shape to chat completions and the flat shape through the Responses bridge,
and answer 400 (or drop under drop_params) on providers that cannot take it.
Guardrails that remove tools now prune them from an allowed_tools choice instead of
deleting fields in place, and an empty tools list is rejected.
This commit is contained in:
mateo-berri 2026-10-28 06:55:05 -07:00
parent 838e51f903
commit 27740a97aa
16 changed files with 654 additions and 135 deletions

View file

@ -29,8 +29,8 @@ from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
ALLOWED_TOOL_CHOICE_ADAPTER,
to_responses_allowed_tool_choice,
ALLOWED_TOOL_CHOICE,
responses_allowed_tool_choice,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -753,7 +753,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return choice_type
if choice_type == "allowed_tools":
with suppress(ValidationError):
return to_responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE_ADAPTER.validate_python(tool_choice))
return responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(tool_choice))
if choice_type not in ("function", "custom"):
return tool_choice
if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):

View file

@ -1,83 +1,154 @@
from collections.abc import Mapping
from typing import Final, Literal, TypeAlias
from collections.abc import Mapping, Sequence, Set
from typing import Final, Literal
from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam
from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
from pydantic import BaseModel, ConfigDict, TypeAdapter
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
PROVIDERS_ACCEPTING_ALLOWED_TOOLS: Final = frozenset({"openai", "azure", "litellm_proxy"})
from litellm.exceptions import UnsupportedParamsError
PROVIDERS_ACCEPTING_ALLOWED_TOOLS: Final = frozenset({"openai", "azure", "azure_ai", "litellm_proxy"})
class _AllowedTools(BaseModel):
model_config = ConfigDict(frozen=True)
model_config = ConfigDict(frozen=True, extra="allow")
mode: Literal["auto", "required"]
tools: tuple[Mapping[str, object], ...]
tools: tuple[Mapping[str, JsonValue], ...] = Field(min_length=1)
class _ChatAllowedToolChoice(BaseModel):
model_config = ConfigDict(frozen=True)
class ChatAllowedToolChoice(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
type: Literal["allowed_tools"]
allowed_tools: _AllowedTools
class _ResponsesAllowedToolChoice(BaseModel):
model_config = ConfigDict(frozen=True)
class ResponsesAllowedToolChoice(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
type: Literal["allowed_tools"]
mode: Literal["auto", "required"]
tools: tuple[Mapping[str, object], ...]
tools: tuple[Mapping[str, JsonValue], ...] = Field(min_length=1)
AllowedToolChoice: TypeAlias = _ChatAllowedToolChoice | _ResponsesAllowedToolChoice
class _AllowedToolChoiceRequest(BaseModel):
model_config = ConfigDict(frozen=True)
ALLOWED_TOOL_CHOICE_ADAPTER: Final[TypeAdapter[AllowedToolChoice]] = TypeAdapter(AllowedToolChoice)
tool_choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice
def _allowed_tools(choice: AllowedToolChoice) -> _AllowedTools:
ALLOWED_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolChoice | ResponsesAllowedToolChoice]] = TypeAdapter(
ChatAllowedToolChoice | ResponsesAllowedToolChoice
)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, JsonValue])
def _json_choice(choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice) -> Mapping[str, JsonValue]:
return _JSON_OBJECT.validate_python(choice.model_dump(mode="json"))
def _chat_tool(tool: JsonValue) -> JsonValue:
match tool:
case {"type": "function", "name": str() as name, **fields}:
return {**fields, "type": "function", "function": {"name": name}}
case {"type": "custom", "name": str() as name, **fields}:
return {**fields, "type": "custom", "custom": {"name": name}}
case _:
return tool
def _responses_tool(tool: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
match tool:
case {"type": "function", "function": {"name": str() as name, **inner_fields}, **fields}:
return {**fields, **inner_fields, "type": "function", "name": name}
case {"type": "custom", "custom": {"name": str() as name, **inner_fields}, **fields}:
return {**fields, **inner_fields, "type": "custom", "name": name}
case _:
return tool
def chat_allowed_tool_choice(
allowed_tool_choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice,
) -> Mapping[str, JsonValue]:
choice: Final = _json_choice(allowed_tool_choice)
match choice:
case _ChatAllowedToolChoice(allowed_tools=allowed_tools):
return allowed_tools
case _ResponsesAllowedToolChoice(mode=mode, tools=tools):
return _AllowedTools(mode=mode, tools=tools)
def _chat_tool(tool: Mapping[str, object]) -> Mapping[str, object]:
match tool:
case {"type": "function", "name": str() as function_name}:
return {"type": "function", "function": {"name": function_name}}
case {"type": "custom", "name": str() as custom_name}:
return {"type": "custom", "custom": {"name": custom_name}}
case {"type": choice_type, "mode": mode, "tools": list() as tools, **allowed_tools_fields}:
return {
"type": choice_type,
"allowed_tools": {**allowed_tools_fields, "mode": mode, "tools": [_chat_tool(tool) for tool in tools]},
}
case _:
return tool
return choice
def _responses_tool(tool: Mapping[str, object]) -> Mapping[str, object]:
match tool:
case {"type": "function", "function": {"name": str() as function_name}}:
return {"type": "function", "name": function_name}
case {"type": "custom", "custom": {"name": str() as custom_name}}:
return {"type": "custom", "name": custom_name}
def responses_allowed_tool_choice(
allowed_tool_choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice,
) -> ToolChoiceAllowedParam:
allowed_tools: Final = (
allowed_tool_choice.allowed_tools
if isinstance(allowed_tool_choice, ChatAllowedToolChoice)
else allowed_tool_choice
)
return {
"type": "allowed_tools",
"mode": allowed_tools.mode,
"tools": [{**_responses_tool(tool)} for tool in allowed_tools.tools],
}
def _is_allowed_tools_choice(tool_choice: object) -> bool:
match tool_choice:
case {"type": "allowed_tools"}:
return True
case _:
return tool
return False
def to_chat_allowed_tool_choice(choice: AllowedToolChoice) -> ChatCompletionAllowedToolChoiceParam:
allowed_tools: Final = _allowed_tools(choice)
return ChatCompletionAllowedToolChoiceParam(
type="allowed_tools",
allowed_tools=ChatCompletionAllowedToolsParam(
mode=allowed_tools.mode, tools=[dict(_chat_tool(tool)) for tool in allowed_tools.tools]
),
def allowed_tool_choice_dropped(
tool_choice: object, *, custom_llm_provider: str, model: str, drop_params: bool
) -> bool:
if not _is_allowed_tools_choice(tool_choice) or custom_llm_provider in PROVIDERS_ACCEPTING_ALLOWED_TOOLS:
return False
if drop_params:
return True
raise UnsupportedParamsError(
message=f"{custom_llm_provider} does not support an allowed_tools tool_choice, for model={model}. To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
model=model,
llm_provider=custom_llm_provider,
)
def to_responses_allowed_tool_choice(choice: AllowedToolChoice) -> ToolChoiceAllowedParam:
allowed_tools: Final = _allowed_tools(choice)
return ToolChoiceAllowedParam(
type="allowed_tools",
mode=allowed_tools.mode,
tools=[dict(_responses_tool(tool)) for tool in allowed_tools.tools],
)
def _function_name(tool: JsonValue) -> str | None:
match tool:
case {"type": "function", "function": {"name": str() as name}} | {"type": "function", "name": str() as name}:
return name
case _:
return None
def _allowed_tools_list(choice: Mapping[str, JsonValue]) -> Sequence[JsonValue]:
match choice:
case {"allowed_tools": {"tools": list() as tools}} | {"tools": list() as tools}:
return tools
case _:
return []
def _with_allowed_tools(choice: Mapping[str, JsonValue], tools: Sequence[JsonValue]) -> JsonValue:
match choice:
case {"allowed_tools": dict() as allowed_tools}:
return {**choice, "allowed_tools": {**allowed_tools, "tools": [*tools]}}
case _:
return {**choice, "tools": [*tools]}
def pruned_allowed_tool_choice(request: object, removed_function_names: Set[str]) -> JsonValue | None:
try:
choice: Final = _json_choice(_AllowedToolChoiceRequest.model_validate(request).tool_choice)
except ValidationError:
return None
tools: Final = _allowed_tools_list(choice)
kept: Final = [tool for tool in tools if _function_name(tool) not in removed_function_names]
if len(kept) == len(tools):
return None
return _with_allowed_tools(choice, kept) if kept else "none"

View file

@ -1,7 +1,7 @@
from __future__ import annotations
import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -294,6 +294,11 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]:
kept_names: Final = frozenset(filter(None, map(openai_tool_name, kept_tools)))
return frozenset(filter(None, map(openai_tool_name, request_tools))) - kept_names
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))

View file

@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
@ -28,6 +29,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.completion_extras.litellm_responses_transformation.transformation import (
replayed_tool_search_tool_names,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -43,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
openai_tool_name,
removed_tool_names,
role_out_of_guardrail_scope,
scoped_structured_message_indices,
stream_item_field,
@ -65,6 +68,8 @@ from litellm.types.utils import (
StreamingChoices,
)
_REQUEST_DATA: Final = TypeAdapter(object)
if TYPE_CHECKING:
from fastapi import HTTPException
@ -76,6 +81,17 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
def _write_guardrailed_tools(request: object, request_tools: Sequence[ChatCompletionToolParam]) -> None:
pruned_tool_choice: Final = pruned_allowed_tool_choice(
request, removed_tool_names(list_field(request, "tools"), request_tools)
)
if not isinstance(request, dict):
return
request["tools"] = request_tools # rebind-ok: in-place request rewrite
if pruned_tool_choice is not None:
request["tool_choice"] = pruned_tool_choice # rebind-ok: in-place request rewrite
class OpenAIChatCompletionsHandler(BaseTranslation):
"""
Handler for processing OpenAI chat completions messages with guardrails.
@ -176,14 +192,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", [])
guardrailed_tools: Final = guardrailed_inputs.get("tools")
if guardrailed_tools is not None:
data["tools"] = (
merge_returned_tools_into_request_tools(
request_tools=tools,
returned_tools=guardrailed_tools,
tool_name=openai_tool_name,
)
if scan_only_tool_results
else guardrailed_tools
_write_guardrailed_tools(
_REQUEST_DATA.validate_python(data),
(
merge_returned_tools_into_request_tools(
request_tools=tools,
returned_tools=guardrailed_tools,
tool_name=openai_tool_name,
)
if scan_only_tool_results
else guardrailed_tools
),
)
guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages")

View file

@ -47,6 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
OpenAiResponsesToChatCompletionStreamIterator,
tool_call_dict_from_output_item,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -58,6 +59,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
removed_tool_names,
role_out_of_guardrail_scope,
scoped_structured_message_indices,
stream_item_field,
@ -117,6 +119,7 @@ if TYPE_CHECKING:
_COMPACTION_SUMMARY_POSITION: Final = -1
_REQUEST_DATA: Final = TypeAdapter(object)
def _output_compaction_summary(item: object) -> str | None:
@ -817,9 +820,13 @@ class OpenAIResponsesHandler(BaseTranslation):
) -> None:
if guardrailed_tools is None:
return
data["tools"] = list( # rebind-ok: in-place request rewrite
merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools)
merged_tools: Final = list(merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools))
data["tools"] = merged_tools # rebind-ok: in-place request rewrite
pruned_tool_choice: Final = pruned_allowed_tool_choice(
_REQUEST_DATA.validate_python(data), removed_tool_names(original_tools, merged_tools)
)
if pruned_tool_choice is not None:
data["tool_choice"] = pruned_tool_choice # rebind-ok: in-place request rewrite
def _extract_input_text_and_images(
self,

View file

@ -69,6 +69,7 @@ from litellm.constants import (
)
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.allowed_tools_tool_choice import allowed_tool_choice_dropped
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
from litellm.litellm_core_utils.audio_utils.utils import (
calculate_request_duration,
@ -5709,7 +5710,7 @@ def completion(
),
)
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
drop_unsupported_reasoning: Final = (
drops_unsupported_params: Final = (
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
)
reasoning_model_group: Final = model_group_from_metadata(
@ -5724,7 +5725,7 @@ def completion(
custom_llm_provider=custom_llm_provider,
deployment_model_info=routing_kwargs.get("model_info"),
model_group=reasoning_model_group,
drop_params=drop_unsupported_reasoning,
drop_params=drops_unsupported_params,
)
resolved_reasoning_effort: Final = chat_reasoning.effort.value
resolved_output_config: Final = chat_reasoning.output_config
@ -5859,7 +5860,16 @@ def completion(
"response_format": response_format,
"seed": seed,
"tools": tools,
"tool_choice": tool_choice,
"tool_choice": (
None
if allowed_tool_choice_dropped(
_OPAQUE_VALUE_ADAPTER.validate_python(tool_choice),
custom_llm_provider=custom_llm_provider,
model=model,
drop_params=drops_unsupported_params,
)
else tool_choice
),
"max_retries": max_retries,
"logprobs": logprobs,
"top_logprobs": top_logprobs,

View file

@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
@ -688,10 +689,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tool_names))
# Create a mapping of tool_use_id to error result
error_tool_names: Final = set()
for tool_use in denied_tool_names:
error_tool_names.add(tool_use)
error_tool_names: Final = frozenset(denied_tool_names)
tools: Final[list[ChatCompletionToolParam] | None] = data.get("tools")
if tools is not None:
@ -712,14 +710,15 @@ class ToolPermissionGuardrail(CustomGuardrail):
named_tool_choice: Final = self._get_named_tool_choice(data)
if named_tool_choice in error_tool_names:
data["tool_choice"] = "none"
pruned_tool_choice: Final = pruned_allowed_tool_choice(_REQUEST_DATA.validate_python(data), error_tool_names)
if pruned_tool_choice is not None:
data["tool_choice"] = pruned_tool_choice # rebind-ok: rewrite mode edits the request in place
named_function_call: Final = self._get_named_function_call(data)
if named_function_call in error_tool_names:
data["function_call"] = "none"
messages: Final = without_tool_search_items_loading(
_REQUEST_DATA.validate_python(data), frozenset(denied_tool_names)
)
messages: Final = without_tool_search_items_loading(_REQUEST_DATA.validate_python(data), error_tool_names)
if messages is not None:
data["messages"] = messages # rebind-ok: rewrite mode edits the request in place

View file

@ -9,6 +9,7 @@ from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping,
from typing import TYPE_CHECKING, Final, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -18,10 +19,13 @@ from litellm.constants import (
DEFAULT_MCP_SEMANTIC_FILTER_TOP_K,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
from litellm.proxy.mcp_server.semantic_tool_filter import (
SemanticToolFilterContextWindowError,
)
_REQUEST_DATA: Final = TypeAdapter(object)
if TYPE_CHECKING:
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
@ -332,6 +336,12 @@ class SemanticToolFilterHook(CustomLogger):
selected_tool_names: Final = self._selected_tool_names(filtered_expanded_tools)
narrowed_tools: Final = await self._narrow_mcp_references(tools, selected_tool_names)
data["tools"] = narrowed_tools
pruned_tool_choice: Final = pruned_allowed_tool_choice(
_REQUEST_DATA.validate_python(data),
frozenset(self._selected_tool_names(expanded_tools)) - frozenset(selected_tool_names),
)
if pruned_tool_choice is not None:
data["tool_choice"] = pruned_tool_choice # rebind-ok: in-place request rewrite
self._emit_filter_metadata_safe(
data=data,
mcp_tools=expanded_tools,
@ -411,6 +421,12 @@ class SemanticToolFilterHook(CustomLogger):
filtered_tools.append(t)
data["tools"] = filtered_tools
filtered_tool_choice: Final = pruned_allowed_tool_choice(
_REQUEST_DATA.validate_python(data),
frozenset(self._selected_tool_names(mcp_tools)) - filtered_mcp_names,
)
if filtered_tool_choice is not None:
data["tool_choice"] = filtered_tool_choice # rebind-ok: in-place request rewrite
self._emit_filter_metadata_safe(
data=data,

View file

@ -82,11 +82,7 @@ from litellm.constants import (
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
ALLOWED_TOOL_CHOICE_ADAPTER,
PROVIDERS_ACCEPTING_ALLOWED_TOOLS,
to_chat_allowed_tool_choice,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import ALLOWED_TOOL_CHOICE, chat_allowed_tool_choice
from litellm.litellm_core_utils.core_helpers import (
bind_budget_reservation_to_callbacks,
normalize_drop_params,
@ -5107,17 +5103,6 @@ def get_optional_params(
_check_valid_arg(
supported_params=supported_params or [],
)
match non_default_params:
case {"tool_choice": {"type": "allowed_tools"}} if custom_llm_provider not in PROVIDERS_ACCEPTING_ALLOWED_TOOLS:
if litellm.drop_params is not True and drop_params is not True:
raise UnsupportedParamsError(
message=f"{custom_llm_provider} does not support an allowed_tools tool_choice, for model={model}. To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
model=model,
llm_provider=custom_llm_provider,
)
del non_default_params["tool_choice"]
case dict():
pass
## raise exception if provider doesn't support passed in param
if custom_llm_provider == "anthropic":
## check if unsupported param passed in
@ -9033,7 +9018,7 @@ def validate_chat_completion_tool_choice(
if tool_choice_type == "allowed_tools":
try:
return dict(to_chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE_ADAPTER.validate_python(tool_choice)))
return dict(chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(tool_choice)))
except ValidationError as exc:
raise BadRequestError(
message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec. {exc}",

View file

@ -0,0 +1,127 @@
from typing import Final
import pytest
from pydantic import ValidationError
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
ALLOWED_TOOL_CHOICE,
chat_allowed_tool_choice,
pruned_allowed_tool_choice,
responses_allowed_tool_choice,
)
CHAT_CHOICE: Final = {
"type": "allowed_tools",
"allowed_tools": {
"mode": "required",
"tools": [
{"type": "function", "function": {"name": "get_weather", "strict": True}, "note": "outer"},
{"type": "custom", "custom": {"name": "apply_patch"}},
{"type": "mcp", "server_label": "deepwiki"},
],
"future_field": 1,
},
}
RESPONSES_CHOICE: Final = {
"type": "allowed_tools",
"mode": "required",
"tools": [
{"type": "function", "name": "get_weather", "strict": True, "note": "outer"},
{"type": "custom", "name": "apply_patch"},
{"type": "mcp", "server_label": "deepwiki"},
],
}
def test_chat_shape_flattens_for_responses_keeping_every_tool_field():
assert responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(CHAT_CHOICE)) == RESPONSES_CHOICE
def test_responses_shape_nests_for_chat_keeping_its_tool_names():
assert chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(RESPONSES_CHOICE)) == {
"type": "allowed_tools",
"allowed_tools": {
"mode": "required",
"tools": [
{"type": "function", "function": {"name": "get_weather"}, "strict": True, "note": "outer"},
{"type": "custom", "custom": {"name": "apply_patch"}},
{"type": "mcp", "server_label": "deepwiki"},
],
},
}
@pytest.mark.parametrize(
("convert", "choice"),
[(chat_allowed_tool_choice, CHAT_CHOICE), (responses_allowed_tool_choice, RESPONSES_CHOICE)],
ids=["chat", "responses"],
)
def test_each_shape_is_unchanged_when_already_in_the_target_shape(convert, choice):
assert convert(ALLOWED_TOOL_CHOICE.validate_python(choice)) == choice
def test_tools_given_as_a_tuple_are_accepted():
choice: Final = {"type": "allowed_tools", "mode": "auto", "tools": ({"type": "function", "name": "get_weather"},)}
assert chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(choice))["allowed_tools"] == {
"mode": "auto",
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
}
@pytest.mark.parametrize(
"choice",
[
{"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}},
{"type": "allowed_tools", "mode": "auto", "tools": []},
{"type": "allowed_tools", "mode": "sometimes", "tools": [{"type": "function", "name": "a"}]},
{"type": "function", "function": {"name": "a"}},
],
)
def test_a_choice_that_is_not_a_valid_allowed_tools_choice_is_rejected(choice):
with pytest.raises(ValidationError):
ALLOWED_TOOL_CHOICE.validate_python(choice)
@pytest.mark.parametrize(
("choice", "expected"),
[
(
CHAT_CHOICE,
{
**CHAT_CHOICE,
"allowed_tools": {
**CHAT_CHOICE["allowed_tools"],
"tools": CHAT_CHOICE["allowed_tools"]["tools"][1:],
},
},
),
(RESPONSES_CHOICE, {**RESPONSES_CHOICE, "tools": RESPONSES_CHOICE["tools"][1:]}),
],
ids=["chat", "responses"],
)
def test_removed_functions_are_pruned_from_the_choice_in_its_own_shape(choice, expected):
assert pruned_allowed_tool_choice({"tool_choice": choice}, {"get_weather"}) == expected
def test_a_choice_left_with_no_tools_becomes_none():
choice: Final = {
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": "get_weather"}],
}
assert pruned_allowed_tool_choice({"tool_choice": choice}, {"get_weather"}) == "none"
@pytest.mark.parametrize(
("request_body", "removed"),
[
({"tool_choice": CHAT_CHOICE}, {"get_time"}),
({"tool_choice": {"type": "function", "function": {"name": "get_weather"}}}, {"get_weather"}),
({"tool_choice": "auto"}, {"get_weather"}),
({}, {"get_weather"}),
],
ids=["other_function_removed", "named_choice", "auto", "no_choice"],
)
def test_the_request_keeps_its_choice_when_nothing_it_allows_is_removed(request_body, removed):
assert pruned_allowed_tool_choice(request_body, removed) is None

View file

@ -80,6 +80,18 @@ class MockGuardrail(CustomGuardrail):
return result
class ToolDroppingGuardrail(CustomGuardrail):
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict,
input_type: Literal["request", "response"],
logging_obj: Optional[Any] = None,
) -> GenericGuardrailAPIInputs:
inputs["tools"] = [tool for tool in inputs.get("tools") or [] if tool["function"]["name"] != "get_time"]
return inputs
class MockCopiedToolCallGuardrail(CustomGuardrail):
"""Mock guardrail that returns copied tool calls instead of mutating inputs."""
@ -2365,3 +2377,45 @@ class TestStreamingScanKey:
for released in (text_only, finished_tool_call):
ended = handler.released_stream_as_ended(released)
assert len(ended) == len(released) and all(a is b for a, b in zip(ended, released, strict=True))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("allowed_names", "expected"),
[
(
["get_weather", "get_time"],
{
"type": "allowed_tools",
"allowed_tools": {
"mode": "required",
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
},
},
),
(["get_time"], "none"),
],
ids=["one_allowed_tool_removed", "every_allowed_tool_removed"],
)
async def test_a_tool_the_guardrail_removes_is_pruned_from_allowed_tools(allowed_names, expected):
data = {
"messages": [{"role": "user", "content": "What time is it?"}],
"tools": [
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}},
{"type": "function", "function": {"name": "get_time", "parameters": {"type": "object"}}},
],
"tool_choice": {
"type": "allowed_tools",
"allowed_tools": {
"mode": "required",
"tools": [{"type": "function", "function": {"name": name}} for name in allowed_names],
},
},
}
result = await OpenAIChatCompletionsHandler().process_input_messages(
data, ToolDroppingGuardrail(guardrail_name="test")
)
assert [tool["function"]["name"] for tool in result["tools"]] == ["get_weather"]
assert result["tool_choice"] == expected

View file

@ -2718,6 +2718,18 @@ class ToolAppendingGuardrail(CustomGuardrail):
return inputs
class ToolDroppingGuardrail(CustomGuardrail):
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict,
input_type: Literal["request", "response"],
logging_obj: Optional[Any] = None,
) -> GenericGuardrailAPIInputs:
inputs["tools"] = [tool for tool in inputs.get("tools") or [] if tool["function"]["name"] != "get_time"]
return inputs
class TestOpenAIResponsesHandlerToolInjection:
"""A tool a guardrail injects must survive the write-back to Responses format."""
@ -2757,6 +2769,40 @@ class TestOpenAIResponsesHandlerToolInjection:
assert "get_weather" in names
assert "injected_tool" in names
@pytest.mark.asyncio
@pytest.mark.parametrize(
("allowed_names", "expected"),
[
(
["get_weather", "get_time"],
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]},
),
(["get_time"], "none"),
],
ids=["one_allowed_tool_removed", "every_allowed_tool_removed"],
)
async def test_a_tool_the_guardrail_removes_is_pruned_from_allowed_tools(self, allowed_names, expected):
data = {
"input": [{"role": "user", "content": "What time is it?", "type": "message"}],
"tools": [
{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}},
{"type": "function", "name": "get_time", "parameters": {"type": "object", "properties": {}}},
],
"tool_choice": {
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": name} for name in allowed_names],
},
"model": "gpt-4",
}
result = await OpenAIResponsesHandler().process_input_messages(
data, ToolDroppingGuardrail(guardrail_name="test")
)
assert [tool["name"] for tool in result["tools"]] == ["get_weather"]
assert result["tool_choice"] == expected
COMPRESSED_MARKER = "[compressed document; retrieve the full text with hash=b573993006976af767214fac]"

View file

@ -751,6 +751,62 @@ class TestToolPermissionGuardrail:
assert new_data["function_call"] == "none"
assert new_data["tool_choice"] == "none"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool_choice", "expected"),
[
(
{
"type": "allowed_tools",
"allowed_tools": {
"mode": "required",
"tools": [
{"type": "function", "function": {"name": "Bash"}},
{"type": "function", "function": {"name": "Read"}},
],
},
},
{
"type": "allowed_tools",
"allowed_tools": {
"mode": "required",
"tools": [{"type": "function", "function": {"name": "Bash"}}],
},
},
),
(
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "Read"}]},
"none",
),
],
ids=["some_allowed_tools_denied", "every_allowed_tool_denied"],
)
async def test_async_pre_call_hook_rewrite_mode_prunes_denied_tools_from_allowed_tools(self, tool_choice, expected):
guardrail = ToolPermissionGuardrail(
guardrail_name="test-tool-permission",
rules=self.test_rules,
default_action="deny",
on_disallowed_action="rewrite",
)
data = {
"tools": [
{"type": "function", "function": {"name": "Bash"}},
{"type": "function", "function": {"name": "Read"}},
],
"tool_choice": tool_choice,
}
with patch.object(guardrail, "should_run_guardrail", return_value=True):
new_data = await guardrail.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(),
cache=DualCache(default_in_memory_ttl=1),
data=data,
call_type="completion",
)
assert isinstance(new_data, dict)
assert new_data["tool_choice"] == expected
@staticmethod
def _tool_search_turn(*loaded_names: str) -> dict:
return {

View file

@ -592,6 +592,67 @@ async def test_semantic_filter_hook_preserves_native_tools():
)
@requires_semantic_router
@pytest.mark.asyncio
async def test_semantic_filter_hook_prunes_filtered_mcp_tools_from_allowed_tools():
from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
from litellm.proxy.mcp_server.semantic_tool_filter import SemanticMCPToolFilter
from litellm.types.utils import Embedding, EmbeddingResponse
def mock_embedding_sync(*args, **kwargs):
return EmbeddingResponse(
data=[Embedding(embedding=[0.1] * 1536, index=0, object="embedding")],
model="text-embedding-3-small",
object="list",
usage={"prompt_tokens": 10, "total_tokens": 10},
)
async def mock_embedding_async(*args, **kwargs):
return mock_embedding_sync()
mock_router = Mock()
mock_router.embedding = mock_embedding_sync
mock_router.aembedding = mock_embedding_async
filter_instance = SemanticMCPToolFilter(
embedding_model="text-embedding-3-small",
litellm_router_instance=mock_router,
top_k=2,
similarity_threshold=0.3,
enabled=True,
)
mcp_tools = [
MCPTool(name=f"mcp_tool_{i}", description=f"MCP tool {i}", inputSchema={"type": "object"}) for i in range(5)
]
filter_instance._build_router(mcp_tools)
native_tool = {"type": "function", "function": {"name": "get_current_weather", "parameters": {"type": "object"}}}
allowed_names = [*(tool.name for tool in mcp_tools), "get_current_weather"]
result = await SemanticToolFilterHook(filter_instance).async_pre_call_hook(
user_api_key_dict=Mock(),
cache=Mock(),
data={
"model": "gpt-4",
"messages": [{"role": "user", "content": "What is the weather?"}],
"tools": [*mcp_tools, native_tool],
"tool_choice": {
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": name} for name in allowed_names],
},
"metadata": {},
},
call_type="completion",
)
kept_mcp_names = [tool.name for tool in result["tools"] if isinstance(tool, MCPTool)]
assert len(kept_mcp_names) < len(mcp_tools)
assert result["tool_choice"] == {
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": name} for name in [*kept_mcp_names, "get_current_weather"]],
}
@requires_semantic_router
@pytest.mark.asyncio
async def test_semantic_filter_hook_all_native_tools():

View file

@ -5503,6 +5503,105 @@ def test_drop_params_keeps_a_dropped_stream_chunk_size_out_of_the_provider_reque
assert sent["model"] == "gpt-4.1-mini"
ALLOWED_TOOLS_REQUEST_TOOLS: Final = [
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}},
{"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}},
]
NESTED_ALLOWED_TOOLS: Final = {
"type": "allowed_tools",
"allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]},
}
FLAT_ALLOWED_TOOLS: Final = {
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": "get_time"}],
}
def _mock_chat_completion(respx_mock: respx.MockRouter) -> respx.Route:
return respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock(
return_value=httpx.Response(
status_code=200,
json={
"id": "chatcmpl-allowed-tools",
"object": "chat.completion",
"created": 1712697600,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
)
@pytest.mark.parametrize("tool_choice", [NESTED_ALLOWED_TOOLS, FLAT_ALLOWED_TOOLS], ids=["nested", "flat"])
@pytest.mark.parametrize(
("model", "provider_kwargs"),
[
("openai/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
("litellm_proxy/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
("azure_ai/DeepSeek-V4-Flash", {"api_base": "http://localhost:12346/models"}),
("azure/gpt-4.1-mini", {"api_base": "http://localhost:12346", "api_version": "2025-04-01-preview"}),
],
)
def test_completion_sends_allowed_tools_nested_to_openai_shaped_providers(
respx_mock: respx.MockRouter, model: str, provider_kwargs: Mapping[str, str], tool_choice: Mapping[str, object]
) -> None:
mock_route: Final = _mock_chat_completion(respx_mock)
litellm.completion(
model=model,
messages=[{"role": "user", "content": "What time is it?"}],
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
tool_choice=tool_choice,
api_key="fake_api_key",
**provider_kwargs,
)
assert mock_route.called
assert json.loads(respx_mock.calls[0].request.content)["tool_choice"] == NESTED_ALLOWED_TOOLS
@pytest.mark.parametrize(
"model", ["anthropic/claude-haiku-4-5", "gemini/gemini-2.5-flash", "ollama/llama3", "deepseek/deepseek-chat"]
)
def test_completion_rejects_allowed_tools_a_provider_cannot_take(model: str) -> None:
with pytest.raises(
litellm.UnsupportedParamsError, match="does not support an allowed_tools tool_choice"
) as exc_info:
litellm.completion(
model=model,
messages=[{"role": "user", "content": "What time is it?"}],
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
tool_choice=NESTED_ALLOWED_TOOLS,
mock_response="hi",
)
assert exc_info.value.status_code == 400
@pytest.mark.parametrize("drop", ["per_call", "global"])
def test_drop_params_sends_tools_without_an_allowed_tools_choice_a_provider_cannot_take(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, drop: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", drop == "global")
mock_route: Final = _mock_chat_completion(respx_mock)
litellm.completion(
model="deepseek/deepseek-chat",
messages=[{"role": "user", "content": "What time is it?"}],
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
tool_choice=FLAT_ALLOWED_TOOLS,
api_base="http://localhost:12346/v1",
api_key="fake_api_key",
drop_params=drop == "per_call",
)
assert mock_route.called
sent: Final = json.loads(respx_mock.calls[0].request.content)
assert "tool_choice" not in sent, sent
assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_weather", "get_time"]
@pytest.mark.asyncio
async def test_global_drop_params_drops_an_invalid_stream_chunk_size_on_acompletion(
monkeypatch: pytest.MonkeyPatch,

View file

@ -8309,6 +8309,8 @@ def test_validate_tool_choice_allowed_tools_takes_the_chat_shape(tool_choice):
{"type": "allowed_tools", "mode": "required"},
{"type": "allowed_tools", "allowed_tools": {"mode": "sometimes", "tools": []}},
{"type": "allowed_tools", "allowed_tools": "get_weather", "mode": "auto"},
{"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}},
{"type": "allowed_tools", "mode": "auto", "tools": []},
],
)
def test_validate_tool_choice_malformed_allowed_tools_is_a_400(tool_choice):
@ -8317,53 +8319,15 @@ def test_validate_tool_choice_malformed_allowed_tools_is_a_400(tool_choice):
assert exc_info.value.status_code == 400
ALLOWED_TOOLS_REQUEST_TOOLS: Final = [
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}
]
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.parametrize(("model", "custom_llm_provider"), [("gpt-5.5", "openai"), ("gpt-5.5", "azure")])
def test_get_optional_params_keeps_allowed_tools_for_openai_shaped_providers(model, custom_llm_provider):
optional_params = get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
def test_get_optional_params_leaves_allowed_tools_to_its_caller():
"""Direct callers like Bedrock batch build provider params here, so the provider check stays in completion()."""
optional_params: Final = get_optional_params(
model="anthropic.claude-haiku-4-5-20251001-v1:0",
custom_llm_provider="bedrock",
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
tool_choice=CHAT_ALLOWED_TOOLS,
)
assert optional_params["tool_choice"] == CHAT_ALLOWED_TOOLS
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), ("deepseek-chat", "deepseek")],
)
def test_get_optional_params_rejects_allowed_tools_a_provider_cannot_take(model, custom_llm_provider):
with pytest.raises(litellm.UnsupportedParamsError, match="does not support an allowed_tools tool_choice") as exc_info:
get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
tool_choice=CHAT_ALLOWED_TOOLS,
)
assert exc_info.value.status_code == 400
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), ("deepseek-chat", "deepseek")],
)
def test_get_optional_params_drops_allowed_tools_under_drop_params(model, custom_llm_provider):
optional_params = get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
tool_choice=CHAT_ALLOWED_TOOLS,
drop_params=True,
)
assert "tool_choice" not in optional_params
assert optional_params["tools"]