mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(tool_choice): accept both allowed_tools shapes and gate unsupported providers
Validate the chat and Responses allowed_tools shapes with pydantic models, send the nested shape to chat completions and the flat shape through the Responses bridge, and answer 400 (or drop under drop_params) on providers that cannot take it. Guardrails that remove tools now prune them from an allowed_tools choice instead of deleting fields in place, and an empty tools list is rejected.
This commit is contained in:
parent
838e51f903
commit
27740a97aa
16 changed files with 654 additions and 135 deletions
|
|
@ -29,8 +29,8 @@ from litellm import ModelResponse
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
ALLOWED_TOOL_CHOICE_ADAPTER,
|
||||
to_responses_allowed_tool_choice,
|
||||
ALLOWED_TOOL_CHOICE,
|
||||
responses_allowed_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -753,7 +753,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
return choice_type
|
||||
if choice_type == "allowed_tools":
|
||||
with suppress(ValidationError):
|
||||
return to_responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE_ADAPTER.validate_python(tool_choice))
|
||||
return responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(tool_choice))
|
||||
if choice_type not in ("function", "custom"):
|
||||
return tool_choice
|
||||
if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
|
||||
|
|
|
|||
|
|
@ -1,83 +1,154 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from collections.abc import Mapping, Sequence, Set
|
||||
from typing import Final, Literal
|
||||
|
||||
from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam
|
||||
from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam
|
||||
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
PROVIDERS_ACCEPTING_ALLOWED_TOOLS: Final = frozenset({"openai", "azure", "litellm_proxy"})
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
|
||||
PROVIDERS_ACCEPTING_ALLOWED_TOOLS: Final = frozenset({"openai", "azure", "azure_ai", "litellm_proxy"})
|
||||
|
||||
|
||||
class _AllowedTools(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
mode: Literal["auto", "required"]
|
||||
tools: tuple[Mapping[str, object], ...]
|
||||
tools: tuple[Mapping[str, JsonValue], ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class _ChatAllowedToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
class ChatAllowedToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
type: Literal["allowed_tools"]
|
||||
allowed_tools: _AllowedTools
|
||||
|
||||
|
||||
class _ResponsesAllowedToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
class ResponsesAllowedToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
type: Literal["allowed_tools"]
|
||||
mode: Literal["auto", "required"]
|
||||
tools: tuple[Mapping[str, object], ...]
|
||||
tools: tuple[Mapping[str, JsonValue], ...] = Field(min_length=1)
|
||||
|
||||
|
||||
AllowedToolChoice: TypeAlias = _ChatAllowedToolChoice | _ResponsesAllowedToolChoice
|
||||
class _AllowedToolChoiceRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
ALLOWED_TOOL_CHOICE_ADAPTER: Final[TypeAdapter[AllowedToolChoice]] = TypeAdapter(AllowedToolChoice)
|
||||
tool_choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice
|
||||
|
||||
|
||||
def _allowed_tools(choice: AllowedToolChoice) -> _AllowedTools:
|
||||
ALLOWED_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolChoice | ResponsesAllowedToolChoice]] = TypeAdapter(
|
||||
ChatAllowedToolChoice | ResponsesAllowedToolChoice
|
||||
)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, JsonValue])
|
||||
|
||||
|
||||
def _json_choice(choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice) -> Mapping[str, JsonValue]:
|
||||
return _JSON_OBJECT.validate_python(choice.model_dump(mode="json"))
|
||||
|
||||
|
||||
def _chat_tool(tool: JsonValue) -> JsonValue:
|
||||
match tool:
|
||||
case {"type": "function", "name": str() as name, **fields}:
|
||||
return {**fields, "type": "function", "function": {"name": name}}
|
||||
case {"type": "custom", "name": str() as name, **fields}:
|
||||
return {**fields, "type": "custom", "custom": {"name": name}}
|
||||
case _:
|
||||
return tool
|
||||
|
||||
|
||||
def _responses_tool(tool: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
|
||||
match tool:
|
||||
case {"type": "function", "function": {"name": str() as name, **inner_fields}, **fields}:
|
||||
return {**fields, **inner_fields, "type": "function", "name": name}
|
||||
case {"type": "custom", "custom": {"name": str() as name, **inner_fields}, **fields}:
|
||||
return {**fields, **inner_fields, "type": "custom", "name": name}
|
||||
case _:
|
||||
return tool
|
||||
|
||||
|
||||
def chat_allowed_tool_choice(
|
||||
allowed_tool_choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice,
|
||||
) -> Mapping[str, JsonValue]:
|
||||
choice: Final = _json_choice(allowed_tool_choice)
|
||||
match choice:
|
||||
case _ChatAllowedToolChoice(allowed_tools=allowed_tools):
|
||||
return allowed_tools
|
||||
case _ResponsesAllowedToolChoice(mode=mode, tools=tools):
|
||||
return _AllowedTools(mode=mode, tools=tools)
|
||||
|
||||
|
||||
def _chat_tool(tool: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match tool:
|
||||
case {"type": "function", "name": str() as function_name}:
|
||||
return {"type": "function", "function": {"name": function_name}}
|
||||
case {"type": "custom", "name": str() as custom_name}:
|
||||
return {"type": "custom", "custom": {"name": custom_name}}
|
||||
case {"type": choice_type, "mode": mode, "tools": list() as tools, **allowed_tools_fields}:
|
||||
return {
|
||||
"type": choice_type,
|
||||
"allowed_tools": {**allowed_tools_fields, "mode": mode, "tools": [_chat_tool(tool) for tool in tools]},
|
||||
}
|
||||
case _:
|
||||
return tool
|
||||
return choice
|
||||
|
||||
|
||||
def _responses_tool(tool: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match tool:
|
||||
case {"type": "function", "function": {"name": str() as function_name}}:
|
||||
return {"type": "function", "name": function_name}
|
||||
case {"type": "custom", "custom": {"name": str() as custom_name}}:
|
||||
return {"type": "custom", "name": custom_name}
|
||||
def responses_allowed_tool_choice(
|
||||
allowed_tool_choice: ChatAllowedToolChoice | ResponsesAllowedToolChoice,
|
||||
) -> ToolChoiceAllowedParam:
|
||||
allowed_tools: Final = (
|
||||
allowed_tool_choice.allowed_tools
|
||||
if isinstance(allowed_tool_choice, ChatAllowedToolChoice)
|
||||
else allowed_tool_choice
|
||||
)
|
||||
return {
|
||||
"type": "allowed_tools",
|
||||
"mode": allowed_tools.mode,
|
||||
"tools": [{**_responses_tool(tool)} for tool in allowed_tools.tools],
|
||||
}
|
||||
|
||||
|
||||
def _is_allowed_tools_choice(tool_choice: object) -> bool:
|
||||
match tool_choice:
|
||||
case {"type": "allowed_tools"}:
|
||||
return True
|
||||
case _:
|
||||
return tool
|
||||
return False
|
||||
|
||||
|
||||
def to_chat_allowed_tool_choice(choice: AllowedToolChoice) -> ChatCompletionAllowedToolChoiceParam:
|
||||
allowed_tools: Final = _allowed_tools(choice)
|
||||
return ChatCompletionAllowedToolChoiceParam(
|
||||
type="allowed_tools",
|
||||
allowed_tools=ChatCompletionAllowedToolsParam(
|
||||
mode=allowed_tools.mode, tools=[dict(_chat_tool(tool)) for tool in allowed_tools.tools]
|
||||
),
|
||||
def allowed_tool_choice_dropped(
|
||||
tool_choice: object, *, custom_llm_provider: str, model: str, drop_params: bool
|
||||
) -> bool:
|
||||
if not _is_allowed_tools_choice(tool_choice) or custom_llm_provider in PROVIDERS_ACCEPTING_ALLOWED_TOOLS:
|
||||
return False
|
||||
if drop_params:
|
||||
return True
|
||||
raise UnsupportedParamsError(
|
||||
message=f"{custom_llm_provider} does not support an allowed_tools tool_choice, for model={model}. To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def to_responses_allowed_tool_choice(choice: AllowedToolChoice) -> ToolChoiceAllowedParam:
|
||||
allowed_tools: Final = _allowed_tools(choice)
|
||||
return ToolChoiceAllowedParam(
|
||||
type="allowed_tools",
|
||||
mode=allowed_tools.mode,
|
||||
tools=[dict(_responses_tool(tool)) for tool in allowed_tools.tools],
|
||||
)
|
||||
def _function_name(tool: JsonValue) -> str | None:
|
||||
match tool:
|
||||
case {"type": "function", "function": {"name": str() as name}} | {"type": "function", "name": str() as name}:
|
||||
return name
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _allowed_tools_list(choice: Mapping[str, JsonValue]) -> Sequence[JsonValue]:
|
||||
match choice:
|
||||
case {"allowed_tools": {"tools": list() as tools}} | {"tools": list() as tools}:
|
||||
return tools
|
||||
case _:
|
||||
return []
|
||||
|
||||
|
||||
def _with_allowed_tools(choice: Mapping[str, JsonValue], tools: Sequence[JsonValue]) -> JsonValue:
|
||||
match choice:
|
||||
case {"allowed_tools": dict() as allowed_tools}:
|
||||
return {**choice, "allowed_tools": {**allowed_tools, "tools": [*tools]}}
|
||||
case _:
|
||||
return {**choice, "tools": [*tools]}
|
||||
|
||||
|
||||
def pruned_allowed_tool_choice(request: object, removed_function_names: Set[str]) -> JsonValue | None:
|
||||
try:
|
||||
choice: Final = _json_choice(_AllowedToolChoiceRequest.model_validate(request).tool_choice)
|
||||
except ValidationError:
|
||||
return None
|
||||
tools: Final = _allowed_tools_list(choice)
|
||||
kept: Final = [tool for tool in tools if _function_name(tool) not in removed_function_names]
|
||||
if len(kept) == len(tools):
|
||||
return None
|
||||
return _with_allowed_tools(choice, kept) if kept else "none"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
|
@ -294,6 +294,11 @@ def openai_tool_name(tool: object) -> str | None:
|
|||
return flat_name if isinstance(flat_name, str) else None
|
||||
|
||||
|
||||
def removed_tool_names(request_tools: Iterable[object], kept_tools: Iterable[object]) -> frozenset[str]:
|
||||
kept_names: Final = frozenset(filter(None, map(openai_tool_name, kept_tools)))
|
||||
return frozenset(filter(None, map(openai_tool_name, request_tools))) - kept_names
|
||||
|
||||
|
||||
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
|
||||
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Union, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -28,6 +29,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
replayed_tool_search_tool_names,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
|
|
@ -43,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
merge_guardrailed_scoped_messages,
|
||||
merge_returned_tools_into_request_tools,
|
||||
openai_tool_name,
|
||||
removed_tool_names,
|
||||
role_out_of_guardrail_scope,
|
||||
scoped_structured_message_indices,
|
||||
stream_item_field,
|
||||
|
|
@ -65,6 +68,8 @@ from litellm.types.utils import (
|
|||
StreamingChoices,
|
||||
)
|
||||
|
||||
_REQUEST_DATA: Final = TypeAdapter(object)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
|
@ -76,6 +81,17 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
|
||||
def _write_guardrailed_tools(request: object, request_tools: Sequence[ChatCompletionToolParam]) -> None:
|
||||
pruned_tool_choice: Final = pruned_allowed_tool_choice(
|
||||
request, removed_tool_names(list_field(request, "tools"), request_tools)
|
||||
)
|
||||
if not isinstance(request, dict):
|
||||
return
|
||||
request["tools"] = request_tools # rebind-ok: in-place request rewrite
|
||||
if pruned_tool_choice is not None:
|
||||
request["tool_choice"] = pruned_tool_choice # rebind-ok: in-place request rewrite
|
||||
|
||||
|
||||
class OpenAIChatCompletionsHandler(BaseTranslation):
|
||||
"""
|
||||
Handler for processing OpenAI chat completions messages with guardrails.
|
||||
|
|
@ -176,14 +192,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", [])
|
||||
guardrailed_tools: Final = guardrailed_inputs.get("tools")
|
||||
if guardrailed_tools is not None:
|
||||
data["tools"] = (
|
||||
merge_returned_tools_into_request_tools(
|
||||
request_tools=tools,
|
||||
returned_tools=guardrailed_tools,
|
||||
tool_name=openai_tool_name,
|
||||
)
|
||||
if scan_only_tool_results
|
||||
else guardrailed_tools
|
||||
_write_guardrailed_tools(
|
||||
_REQUEST_DATA.validate_python(data),
|
||||
(
|
||||
merge_returned_tools_into_request_tools(
|
||||
request_tools=tools,
|
||||
returned_tools=guardrailed_tools,
|
||||
tool_name=openai_tool_name,
|
||||
)
|
||||
if scan_only_tool_results
|
||||
else guardrailed_tools
|
||||
),
|
||||
)
|
||||
|
||||
guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages")
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
|
|||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
tool_call_dict_from_output_item,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
|
|
@ -58,6 +59,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
effective_skip_system_message_for_guardrail,
|
||||
list_field,
|
||||
merge_guardrailed_scoped_messages,
|
||||
removed_tool_names,
|
||||
role_out_of_guardrail_scope,
|
||||
scoped_structured_message_indices,
|
||||
stream_item_field,
|
||||
|
|
@ -117,6 +119,7 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
_COMPACTION_SUMMARY_POSITION: Final = -1
|
||||
_REQUEST_DATA: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _output_compaction_summary(item: object) -> str | None:
|
||||
|
|
@ -817,9 +820,13 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
) -> None:
|
||||
if guardrailed_tools is None:
|
||||
return
|
||||
data["tools"] = list( # rebind-ok: in-place request rewrite
|
||||
merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools)
|
||||
merged_tools: Final = list(merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools))
|
||||
data["tools"] = merged_tools # rebind-ok: in-place request rewrite
|
||||
pruned_tool_choice: Final = pruned_allowed_tool_choice(
|
||||
_REQUEST_DATA.validate_python(data), removed_tool_names(original_tools, merged_tools)
|
||||
)
|
||||
if pruned_tool_choice is not None:
|
||||
data["tool_choice"] = pruned_tool_choice # rebind-ok: in-place request rewrite
|
||||
|
||||
def _extract_input_text_and_images(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.exceptions import LiteLLMUnknownProvider
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import allowed_tool_choice_dropped
|
||||
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
|
||||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
calculate_request_duration,
|
||||
|
|
@ -5709,7 +5710,7 @@ def completion(
|
|||
),
|
||||
)
|
||||
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
|
||||
drop_unsupported_reasoning: Final = (
|
||||
drops_unsupported_params: Final = (
|
||||
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
|
||||
)
|
||||
reasoning_model_group: Final = model_group_from_metadata(
|
||||
|
|
@ -5724,7 +5725,7 @@ def completion(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=routing_kwargs.get("model_info"),
|
||||
model_group=reasoning_model_group,
|
||||
drop_params=drop_unsupported_reasoning,
|
||||
drop_params=drops_unsupported_params,
|
||||
)
|
||||
resolved_reasoning_effort: Final = chat_reasoning.effort.value
|
||||
resolved_output_config: Final = chat_reasoning.output_config
|
||||
|
|
@ -5859,7 +5860,16 @@ def completion(
|
|||
"response_format": response_format,
|
||||
"seed": seed,
|
||||
"tools": tools,
|
||||
"tool_choice": tool_choice,
|
||||
"tool_choice": (
|
||||
None
|
||||
if allowed_tool_choice_dropped(
|
||||
_OPAQUE_VALUE_ADAPTER.validate_python(tool_choice),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
drop_params=drops_unsupported_params,
|
||||
)
|
||||
else tool_choice
|
||||
),
|
||||
"max_retries": max_retries,
|
||||
"logprobs": logprobs,
|
||||
"top_logprobs": top_logprobs,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
|
|
@ -688,10 +689,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
|
||||
verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tool_names))
|
||||
|
||||
# Create a mapping of tool_use_id to error result
|
||||
error_tool_names: Final = set()
|
||||
for tool_use in denied_tool_names:
|
||||
error_tool_names.add(tool_use)
|
||||
error_tool_names: Final = frozenset(denied_tool_names)
|
||||
|
||||
tools: Final[list[ChatCompletionToolParam] | None] = data.get("tools")
|
||||
if tools is not None:
|
||||
|
|
@ -712,14 +710,15 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
named_tool_choice: Final = self._get_named_tool_choice(data)
|
||||
if named_tool_choice in error_tool_names:
|
||||
data["tool_choice"] = "none"
|
||||
pruned_tool_choice: Final = pruned_allowed_tool_choice(_REQUEST_DATA.validate_python(data), error_tool_names)
|
||||
if pruned_tool_choice is not None:
|
||||
data["tool_choice"] = pruned_tool_choice # rebind-ok: rewrite mode edits the request in place
|
||||
|
||||
named_function_call: Final = self._get_named_function_call(data)
|
||||
if named_function_call in error_tool_names:
|
||||
data["function_call"] = "none"
|
||||
|
||||
messages: Final = without_tool_search_items_loading(
|
||||
_REQUEST_DATA.validate_python(data), frozenset(denied_tool_names)
|
||||
)
|
||||
messages: Final = without_tool_search_items_loading(_REQUEST_DATA.validate_python(data), error_tool_names)
|
||||
if messages is not None:
|
||||
data["messages"] = messages # rebind-ok: rewrite mode edits the request in place
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping,
|
|||
from typing import TYPE_CHECKING, Final, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -18,10 +19,13 @@ from litellm.constants import (
|
|||
DEFAULT_MCP_SEMANTIC_FILTER_TOP_K,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import pruned_allowed_tool_choice
|
||||
from litellm.proxy.mcp_server.semantic_tool_filter import (
|
||||
SemanticToolFilterContextWindowError,
|
||||
)
|
||||
|
||||
_REQUEST_DATA: Final = TypeAdapter(object)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
|
@ -332,6 +336,12 @@ class SemanticToolFilterHook(CustomLogger):
|
|||
selected_tool_names: Final = self._selected_tool_names(filtered_expanded_tools)
|
||||
narrowed_tools: Final = await self._narrow_mcp_references(tools, selected_tool_names)
|
||||
data["tools"] = narrowed_tools
|
||||
pruned_tool_choice: Final = pruned_allowed_tool_choice(
|
||||
_REQUEST_DATA.validate_python(data),
|
||||
frozenset(self._selected_tool_names(expanded_tools)) - frozenset(selected_tool_names),
|
||||
)
|
||||
if pruned_tool_choice is not None:
|
||||
data["tool_choice"] = pruned_tool_choice # rebind-ok: in-place request rewrite
|
||||
self._emit_filter_metadata_safe(
|
||||
data=data,
|
||||
mcp_tools=expanded_tools,
|
||||
|
|
@ -411,6 +421,12 @@ class SemanticToolFilterHook(CustomLogger):
|
|||
filtered_tools.append(t)
|
||||
|
||||
data["tools"] = filtered_tools
|
||||
filtered_tool_choice: Final = pruned_allowed_tool_choice(
|
||||
_REQUEST_DATA.validate_python(data),
|
||||
frozenset(self._selected_tool_names(mcp_tools)) - filtered_mcp_names,
|
||||
)
|
||||
if filtered_tool_choice is not None:
|
||||
data["tool_choice"] = filtered_tool_choice # rebind-ok: in-place request rewrite
|
||||
|
||||
self._emit_filter_metadata_safe(
|
||||
data=data,
|
||||
|
|
|
|||
|
|
@ -82,11 +82,7 @@ from litellm.constants import (
|
|||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
ALLOWED_TOOL_CHOICE_ADAPTER,
|
||||
PROVIDERS_ACCEPTING_ALLOWED_TOOLS,
|
||||
to_chat_allowed_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import ALLOWED_TOOL_CHOICE, chat_allowed_tool_choice
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
bind_budget_reservation_to_callbacks,
|
||||
normalize_drop_params,
|
||||
|
|
@ -5107,17 +5103,6 @@ def get_optional_params(
|
|||
_check_valid_arg(
|
||||
supported_params=supported_params or [],
|
||||
)
|
||||
match non_default_params:
|
||||
case {"tool_choice": {"type": "allowed_tools"}} if custom_llm_provider not in PROVIDERS_ACCEPTING_ALLOWED_TOOLS:
|
||||
if litellm.drop_params is not True and drop_params is not True:
|
||||
raise UnsupportedParamsError(
|
||||
message=f"{custom_llm_provider} does not support an allowed_tools tool_choice, for model={model}. To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
del non_default_params["tool_choice"]
|
||||
case dict():
|
||||
pass
|
||||
## raise exception if provider doesn't support passed in param
|
||||
if custom_llm_provider == "anthropic":
|
||||
## check if unsupported param passed in
|
||||
|
|
@ -9033,7 +9018,7 @@ def validate_chat_completion_tool_choice(
|
|||
|
||||
if tool_choice_type == "allowed_tools":
|
||||
try:
|
||||
return dict(to_chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE_ADAPTER.validate_python(tool_choice)))
|
||||
return dict(chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(tool_choice)))
|
||||
except ValidationError as exc:
|
||||
raise BadRequestError(
|
||||
message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec. {exc}",
|
||||
|
|
|
|||
127
tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py
Normal file
127
tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
ALLOWED_TOOL_CHOICE,
|
||||
chat_allowed_tool_choice,
|
||||
pruned_allowed_tool_choice,
|
||||
responses_allowed_tool_choice,
|
||||
)
|
||||
|
||||
CHAT_CHOICE: Final = {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_weather", "strict": True}, "note": "outer"},
|
||||
{"type": "custom", "custom": {"name": "apply_patch"}},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
"future_field": 1,
|
||||
},
|
||||
}
|
||||
|
||||
RESPONSES_CHOICE: Final = {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "name": "get_weather", "strict": True, "note": "outer"},
|
||||
{"type": "custom", "name": "apply_patch"},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_chat_shape_flattens_for_responses_keeping_every_tool_field():
|
||||
assert responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(CHAT_CHOICE)) == RESPONSES_CHOICE
|
||||
|
||||
|
||||
def test_responses_shape_nests_for_chat_keeping_its_tool_names():
|
||||
assert chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(RESPONSES_CHOICE)) == {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_weather"}, "strict": True, "note": "outer"},
|
||||
{"type": "custom", "custom": {"name": "apply_patch"}},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("convert", "choice"),
|
||||
[(chat_allowed_tool_choice, CHAT_CHOICE), (responses_allowed_tool_choice, RESPONSES_CHOICE)],
|
||||
ids=["chat", "responses"],
|
||||
)
|
||||
def test_each_shape_is_unchanged_when_already_in_the_target_shape(convert, choice):
|
||||
assert convert(ALLOWED_TOOL_CHOICE.validate_python(choice)) == choice
|
||||
|
||||
|
||||
def test_tools_given_as_a_tuple_are_accepted():
|
||||
choice: Final = {"type": "allowed_tools", "mode": "auto", "tools": ({"type": "function", "name": "get_weather"},)}
|
||||
assert chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE.validate_python(choice))["allowed_tools"] == {
|
||||
"mode": "auto",
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"choice",
|
||||
[
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": []},
|
||||
{"type": "allowed_tools", "mode": "sometimes", "tools": [{"type": "function", "name": "a"}]},
|
||||
{"type": "function", "function": {"name": "a"}},
|
||||
],
|
||||
)
|
||||
def test_a_choice_that_is_not_a_valid_allowed_tools_choice_is_rejected(choice):
|
||||
with pytest.raises(ValidationError):
|
||||
ALLOWED_TOOL_CHOICE.validate_python(choice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("choice", "expected"),
|
||||
[
|
||||
(
|
||||
CHAT_CHOICE,
|
||||
{
|
||||
**CHAT_CHOICE,
|
||||
"allowed_tools": {
|
||||
**CHAT_CHOICE["allowed_tools"],
|
||||
"tools": CHAT_CHOICE["allowed_tools"]["tools"][1:],
|
||||
},
|
||||
},
|
||||
),
|
||||
(RESPONSES_CHOICE, {**RESPONSES_CHOICE, "tools": RESPONSES_CHOICE["tools"][1:]}),
|
||||
],
|
||||
ids=["chat", "responses"],
|
||||
)
|
||||
def test_removed_functions_are_pruned_from_the_choice_in_its_own_shape(choice, expected):
|
||||
assert pruned_allowed_tool_choice({"tool_choice": choice}, {"get_weather"}) == expected
|
||||
|
||||
|
||||
def test_a_choice_left_with_no_tools_becomes_none():
|
||||
choice: Final = {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "name": "get_weather"}],
|
||||
}
|
||||
assert pruned_allowed_tool_choice({"tool_choice": choice}, {"get_weather"}) == "none"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_body", "removed"),
|
||||
[
|
||||
({"tool_choice": CHAT_CHOICE}, {"get_time"}),
|
||||
({"tool_choice": {"type": "function", "function": {"name": "get_weather"}}}, {"get_weather"}),
|
||||
({"tool_choice": "auto"}, {"get_weather"}),
|
||||
({}, {"get_weather"}),
|
||||
],
|
||||
ids=["other_function_removed", "named_choice", "auto", "no_choice"],
|
||||
)
|
||||
def test_the_request_keeps_its_choice_when_nothing_it_allows_is_removed(request_body, removed):
|
||||
assert pruned_allowed_tool_choice(request_body, removed) is None
|
||||
|
|
@ -80,6 +80,18 @@ class MockGuardrail(CustomGuardrail):
|
|||
return result
|
||||
|
||||
|
||||
class ToolDroppingGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict,
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: Optional[Any] = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
inputs["tools"] = [tool for tool in inputs.get("tools") or [] if tool["function"]["name"] != "get_time"]
|
||||
return inputs
|
||||
|
||||
|
||||
class MockCopiedToolCallGuardrail(CustomGuardrail):
|
||||
"""Mock guardrail that returns copied tool calls instead of mutating inputs."""
|
||||
|
||||
|
|
@ -2365,3 +2377,45 @@ class TestStreamingScanKey:
|
|||
for released in (text_only, finished_tool_call):
|
||||
ended = handler.released_stream_as_ended(released)
|
||||
assert len(ended) == len(released) and all(a is b for a, b in zip(ended, released, strict=True))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("allowed_names", "expected"),
|
||||
[
|
||||
(
|
||||
["get_weather", "get_time"],
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
|
||||
},
|
||||
},
|
||||
),
|
||||
(["get_time"], "none"),
|
||||
],
|
||||
ids=["one_allowed_tool_removed", "every_allowed_tool_removed"],
|
||||
)
|
||||
async def test_a_tool_the_guardrail_removes_is_pruned_from_allowed_tools(allowed_names, expected):
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "What time is it?"}],
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}},
|
||||
{"type": "function", "function": {"name": "get_time", "parameters": {"type": "object"}}},
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "function": {"name": name}} for name in allowed_names],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result = await OpenAIChatCompletionsHandler().process_input_messages(
|
||||
data, ToolDroppingGuardrail(guardrail_name="test")
|
||||
)
|
||||
|
||||
assert [tool["function"]["name"] for tool in result["tools"]] == ["get_weather"]
|
||||
assert result["tool_choice"] == expected
|
||||
|
|
|
|||
|
|
@ -2718,6 +2718,18 @@ class ToolAppendingGuardrail(CustomGuardrail):
|
|||
return inputs
|
||||
|
||||
|
||||
class ToolDroppingGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict,
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: Optional[Any] = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
inputs["tools"] = [tool for tool in inputs.get("tools") or [] if tool["function"]["name"] != "get_time"]
|
||||
return inputs
|
||||
|
||||
|
||||
class TestOpenAIResponsesHandlerToolInjection:
|
||||
"""A tool a guardrail injects must survive the write-back to Responses format."""
|
||||
|
||||
|
|
@ -2757,6 +2769,40 @@ class TestOpenAIResponsesHandlerToolInjection:
|
|||
assert "get_weather" in names
|
||||
assert "injected_tool" in names
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("allowed_names", "expected"),
|
||||
[
|
||||
(
|
||||
["get_weather", "get_time"],
|
||||
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
),
|
||||
(["get_time"], "none"),
|
||||
],
|
||||
ids=["one_allowed_tool_removed", "every_allowed_tool_removed"],
|
||||
)
|
||||
async def test_a_tool_the_guardrail_removes_is_pruned_from_allowed_tools(self, allowed_names, expected):
|
||||
data = {
|
||||
"input": [{"role": "user", "content": "What time is it?", "type": "message"}],
|
||||
"tools": [
|
||||
{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}},
|
||||
{"type": "function", "name": "get_time", "parameters": {"type": "object", "properties": {}}},
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "name": name} for name in allowed_names],
|
||||
},
|
||||
"model": "gpt-4",
|
||||
}
|
||||
|
||||
result = await OpenAIResponsesHandler().process_input_messages(
|
||||
data, ToolDroppingGuardrail(guardrail_name="test")
|
||||
)
|
||||
|
||||
assert [tool["name"] for tool in result["tools"]] == ["get_weather"]
|
||||
assert result["tool_choice"] == expected
|
||||
|
||||
|
||||
COMPRESSED_MARKER = "[compressed document; retrieve the full text with hash=b573993006976af767214fac]"
|
||||
|
||||
|
|
|
|||
|
|
@ -751,6 +751,62 @@ class TestToolPermissionGuardrail:
|
|||
assert new_data["function_call"] == "none"
|
||||
assert new_data["tool_choice"] == "none"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "expected"),
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "Bash"}},
|
||||
{"type": "function", "function": {"name": "Read"}},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "function": {"name": "Bash"}}],
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "Read"}]},
|
||||
"none",
|
||||
),
|
||||
],
|
||||
ids=["some_allowed_tools_denied", "every_allowed_tool_denied"],
|
||||
)
|
||||
async def test_async_pre_call_hook_rewrite_mode_prunes_denied_tools_from_allowed_tools(self, tool_choice, expected):
|
||||
guardrail = ToolPermissionGuardrail(
|
||||
guardrail_name="test-tool-permission",
|
||||
rules=self.test_rules,
|
||||
default_action="deny",
|
||||
on_disallowed_action="rewrite",
|
||||
)
|
||||
data = {
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "Bash"}},
|
||||
{"type": "function", "function": {"name": "Read"}},
|
||||
],
|
||||
"tool_choice": tool_choice,
|
||||
}
|
||||
|
||||
with patch.object(guardrail, "should_run_guardrail", return_value=True):
|
||||
new_data = await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=DualCache(default_in_memory_ttl=1),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert isinstance(new_data, dict)
|
||||
assert new_data["tool_choice"] == expected
|
||||
|
||||
@staticmethod
|
||||
def _tool_search_turn(*loaded_names: str) -> dict:
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -592,6 +592,67 @@ async def test_semantic_filter_hook_preserves_native_tools():
|
|||
)
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_prunes_filtered_mcp_tools_from_allowed_tools():
|
||||
from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
|
||||
from litellm.proxy.mcp_server.semantic_tool_filter import SemanticMCPToolFilter
|
||||
from litellm.types.utils import Embedding, EmbeddingResponse
|
||||
|
||||
def mock_embedding_sync(*args, **kwargs):
|
||||
return EmbeddingResponse(
|
||||
data=[Embedding(embedding=[0.1] * 1536, index=0, object="embedding")],
|
||||
model="text-embedding-3-small",
|
||||
object="list",
|
||||
usage={"prompt_tokens": 10, "total_tokens": 10},
|
||||
)
|
||||
|
||||
async def mock_embedding_async(*args, **kwargs):
|
||||
return mock_embedding_sync()
|
||||
|
||||
mock_router = Mock()
|
||||
mock_router.embedding = mock_embedding_sync
|
||||
mock_router.aembedding = mock_embedding_async
|
||||
filter_instance = SemanticMCPToolFilter(
|
||||
embedding_model="text-embedding-3-small",
|
||||
litellm_router_instance=mock_router,
|
||||
top_k=2,
|
||||
similarity_threshold=0.3,
|
||||
enabled=True,
|
||||
)
|
||||
mcp_tools = [
|
||||
MCPTool(name=f"mcp_tool_{i}", description=f"MCP tool {i}", inputSchema={"type": "object"}) for i in range(5)
|
||||
]
|
||||
filter_instance._build_router(mcp_tools)
|
||||
native_tool = {"type": "function", "function": {"name": "get_current_weather", "parameters": {"type": "object"}}}
|
||||
allowed_names = [*(tool.name for tool in mcp_tools), "get_current_weather"]
|
||||
|
||||
result = await SemanticToolFilterHook(filter_instance).async_pre_call_hook(
|
||||
user_api_key_dict=Mock(),
|
||||
cache=Mock(),
|
||||
data={
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "What is the weather?"}],
|
||||
"tools": [*mcp_tools, native_tool],
|
||||
"tool_choice": {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "name": name} for name in allowed_names],
|
||||
},
|
||||
"metadata": {},
|
||||
},
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
kept_mcp_names = [tool.name for tool in result["tools"] if isinstance(tool, MCPTool)]
|
||||
assert len(kept_mcp_names) < len(mcp_tools)
|
||||
assert result["tool_choice"] == {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "name": name} for name in [*kept_mcp_names, "get_current_weather"]],
|
||||
}
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_all_native_tools():
|
||||
|
|
|
|||
|
|
@ -5503,6 +5503,105 @@ def test_drop_params_keeps_a_dropped_stream_chunk_size_out_of_the_provider_reque
|
|||
assert sent["model"] == "gpt-4.1-mini"
|
||||
|
||||
|
||||
ALLOWED_TOOLS_REQUEST_TOOLS: Final = [
|
||||
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}},
|
||||
{"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}},
|
||||
]
|
||||
NESTED_ALLOWED_TOOLS: Final = {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]},
|
||||
}
|
||||
FLAT_ALLOWED_TOOLS: Final = {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "name": "get_time"}],
|
||||
}
|
||||
|
||||
|
||||
def _mock_chat_completion(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock(
|
||||
return_value=httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-allowed-tools",
|
||||
"object": "chat.completion",
|
||||
"created": 1712697600,
|
||||
"model": "gpt-4.1-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", [NESTED_ALLOWED_TOOLS, FLAT_ALLOWED_TOOLS], ids=["nested", "flat"])
|
||||
@pytest.mark.parametrize(
|
||||
("model", "provider_kwargs"),
|
||||
[
|
||||
("openai/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
|
||||
("litellm_proxy/gpt-4.1-mini", {"api_base": "http://localhost:12346/v1"}),
|
||||
("azure_ai/DeepSeek-V4-Flash", {"api_base": "http://localhost:12346/models"}),
|
||||
("azure/gpt-4.1-mini", {"api_base": "http://localhost:12346", "api_version": "2025-04-01-preview"}),
|
||||
],
|
||||
)
|
||||
def test_completion_sends_allowed_tools_nested_to_openai_shaped_providers(
|
||||
respx_mock: respx.MockRouter, model: str, provider_kwargs: Mapping[str, str], tool_choice: Mapping[str, object]
|
||||
) -> None:
|
||||
mock_route: Final = _mock_chat_completion(respx_mock)
|
||||
|
||||
litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What time is it?"}],
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=tool_choice,
|
||||
api_key="fake_api_key",
|
||||
**provider_kwargs,
|
||||
)
|
||||
|
||||
assert mock_route.called
|
||||
assert json.loads(respx_mock.calls[0].request.content)["tool_choice"] == NESTED_ALLOWED_TOOLS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["anthropic/claude-haiku-4-5", "gemini/gemini-2.5-flash", "ollama/llama3", "deepseek/deepseek-chat"]
|
||||
)
|
||||
def test_completion_rejects_allowed_tools_a_provider_cannot_take(model: str) -> None:
|
||||
with pytest.raises(
|
||||
litellm.UnsupportedParamsError, match="does not support an allowed_tools tool_choice"
|
||||
) as exc_info:
|
||||
litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What time is it?"}],
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=NESTED_ALLOWED_TOOLS,
|
||||
mock_response="hi",
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop", ["per_call", "global"])
|
||||
def test_drop_params_sends_tools_without_an_allowed_tools_choice_a_provider_cannot_take(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, drop: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", drop == "global")
|
||||
mock_route: Final = _mock_chat_completion(respx_mock)
|
||||
|
||||
litellm.completion(
|
||||
model="deepseek/deepseek-chat",
|
||||
messages=[{"role": "user", "content": "What time is it?"}],
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=FLAT_ALLOWED_TOOLS,
|
||||
api_base="http://localhost:12346/v1",
|
||||
api_key="fake_api_key",
|
||||
drop_params=drop == "per_call",
|
||||
)
|
||||
|
||||
assert mock_route.called
|
||||
sent: Final = json.loads(respx_mock.calls[0].request.content)
|
||||
assert "tool_choice" not in sent, sent
|
||||
assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_weather", "get_time"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_drop_params_drops_an_invalid_stream_chunk_size_on_acompletion(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
|
|
@ -8309,6 +8309,8 @@ def test_validate_tool_choice_allowed_tools_takes_the_chat_shape(tool_choice):
|
|||
{"type": "allowed_tools", "mode": "required"},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "sometimes", "tools": []}},
|
||||
{"type": "allowed_tools", "allowed_tools": "get_weather", "mode": "auto"},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": []},
|
||||
],
|
||||
)
|
||||
def test_validate_tool_choice_malformed_allowed_tools_is_a_400(tool_choice):
|
||||
|
|
@ -8317,53 +8319,15 @@ def test_validate_tool_choice_malformed_allowed_tools_is_a_400(tool_choice):
|
|||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
ALLOWED_TOOLS_REQUEST_TOOLS: Final = [
|
||||
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(("model", "custom_llm_provider"), [("gpt-5.5", "openai"), ("gpt-5.5", "azure")])
|
||||
def test_get_optional_params_keeps_allowed_tools_for_openai_shaped_providers(model, custom_llm_provider):
|
||||
optional_params = get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
def test_get_optional_params_leaves_allowed_tools_to_its_caller():
|
||||
"""Direct callers like Bedrock batch build provider params here, so the provider check stays in completion()."""
|
||||
optional_params: Final = get_optional_params(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
custom_llm_provider="bedrock",
|
||||
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
|
||||
tool_choice=CHAT_ALLOWED_TOOLS,
|
||||
)
|
||||
assert optional_params["tool_choice"] == CHAT_ALLOWED_TOOLS
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider"),
|
||||
[("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), ("deepseek-chat", "deepseek")],
|
||||
)
|
||||
def test_get_optional_params_rejects_allowed_tools_a_provider_cannot_take(model, custom_llm_provider):
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="does not support an allowed_tools tool_choice") as exc_info:
|
||||
get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=CHAT_ALLOWED_TOOLS,
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider"),
|
||||
[("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), ("deepseek-chat", "deepseek")],
|
||||
)
|
||||
def test_get_optional_params_drops_allowed_tools_under_drop_params(model, custom_llm_provider):
|
||||
optional_params = get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=CHAT_ALLOWED_TOOLS,
|
||||
drop_params=True,
|
||||
)
|
||||
assert "tool_choice" not in optional_params
|
||||
assert optional_params["tools"]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue