mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(guardrails): check forced, grouped, loaded, and hosted tools by their real type
This commit is contained in:
parent
69dae9682e
commit
14bf22f394
2 changed files with 274 additions and 32 deletions
|
|
@ -1,7 +1,10 @@
|
|||
import json
|
||||
import re
|
||||
from collections.abc import AsyncGenerator, AsyncIterable, Mapping, Sequence
|
||||
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Literal, TypeAlias, TypedDict
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -47,9 +50,13 @@ from litellm.types.utils import (
|
|||
|
||||
GUARDRAIL_NAME: Final = "tool_permission"
|
||||
_RESPONSES_CALL_TYPES: Final = frozenset({"responses", "aresponses", "_aresponses_websocket"})
|
||||
_ANTHROPIC_CALL_TYPES: Final = frozenset({"anthropic_messages", "aanthropic_messages"})
|
||||
_HOSTED_SERVER_NAME_FIELDS: Final = MappingProxyType({"mcp": "server_label", "mcp_toolset": "mcp_server_name"})
|
||||
_DECLARATION_KEYS: Final = ("functionDeclarations", "function_declarations")
|
||||
_NAMED_CHOICE_TYPES: Final = frozenset({"tool", "function", "custom", "allowed_tools", "any", "auto", "none"})
|
||||
_DECODED_ARGUMENTS: Final = TypeAdapter(object)
|
||||
_REQUEST_DATA: Final = TypeAdapter(object)
|
||||
_OPTIONAL_REQUEST_LIST: Final = TypeAdapter(list[object] | None)
|
||||
_OPTIONAL_REQUEST_LIST: Final[TypeAdapter[list[object] | None]] = TypeAdapter(list[object] | None)
|
||||
_ToolTarget: TypeAlias = tuple[str | None, str | None]
|
||||
|
||||
|
||||
|
|
@ -79,7 +86,11 @@ def _request_tool_names(tool: object) -> tuple[str, ...]:
|
|||
_object_mapping(fields.get("custom")) if fields is not None and fields.get("type") == "custom" else None
|
||||
)
|
||||
custom_name: Final = _non_empty_string(custom.get("name")) if custom is not None else None
|
||||
return anthropic_tool_names(tool) + ((custom_name,) if custom_name is not None else ())
|
||||
server_field: Final = _HOSTED_SERVER_NAME_FIELDS.get(str(fields.get("type"))) if fields is not None else None
|
||||
server_name: Final = (
|
||||
_non_empty_string(fields.get(server_field)) if fields is not None and server_field is not None else None
|
||||
)
|
||||
return anthropic_tool_names(tool) + tuple(name for name in (custom_name, server_name) if name is not None)
|
||||
|
||||
|
||||
def _namespace_members(tool: object) -> Sequence[object]:
|
||||
|
|
@ -96,10 +107,57 @@ def _namespace_member_target(namespace: str | None, member: object) -> _ToolTarg
|
|||
return None
|
||||
member_type: Final = _non_empty_string(fields.get("type"))
|
||||
name: Final = _non_empty_string(fields.get("name"))
|
||||
qualified: Final = name is not None and namespace is not None and member_type != "custom"
|
||||
qualified: Final = name is not None and namespace is not None
|
||||
return (f"{namespace}__{name}" if qualified else name, member_type)
|
||||
|
||||
|
||||
def _declaration_target(declaration: object) -> _ToolTarget | None:
|
||||
fields: Final = _object_mapping(declaration)
|
||||
name: Final = _non_empty_string(fields.get("name")) if fields is not None else None
|
||||
return (name, "function") if name is not None else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _MemberGroup:
|
||||
key: str
|
||||
members: Sequence[object]
|
||||
target_of: Callable[[object], _ToolTarget | None]
|
||||
|
||||
|
||||
def _member_group(tool: object) -> _MemberGroup | None:
|
||||
fields: Final = _object_mapping(tool)
|
||||
if fields is None:
|
||||
return None
|
||||
members: Final = _namespace_members(tool)
|
||||
if members:
|
||||
return _MemberGroup("tools", members, partial(_namespace_member_target, _non_empty_string(fields.get("name"))))
|
||||
key: Final = next((key for key in _DECLARATION_KEYS if _object_list(fields.get(key))), None)
|
||||
declarations: Final = _object_list(fields.get(key)) if key is not None else None
|
||||
return _MemberGroup(key, declarations, _declaration_target) if key is not None and declarations else None
|
||||
|
||||
|
||||
def _sole_key(tool: object) -> str | None:
|
||||
fields: Final = _object_mapping(tool)
|
||||
keys: Final = tuple(fields) if fields is not None else ()
|
||||
return keys[0] if len(keys) == 1 and keys[0] not in _DECLARATION_KEYS else None
|
||||
|
||||
|
||||
def _tool_search_output_tools(item: object) -> Sequence[object] | None:
|
||||
fields: Final = _object_mapping(item)
|
||||
return (
|
||||
_object_list(fields.get("tools")) if fields is not None and fields.get("type") == "tool_search_output" else None
|
||||
)
|
||||
|
||||
|
||||
def _input_items(data: object) -> Sequence[object]:
|
||||
fields: Final = _object_mapping(data)
|
||||
return (_object_list(fields.get("input")) if fields is not None else None) or ()
|
||||
|
||||
|
||||
def _loaded_input_tools(data: object) -> tuple[object, ...]:
|
||||
return tuple(chain.from_iterable(_tool_search_output_tools(item) or () for item in _input_items(data)))
|
||||
|
||||
|
||||
def _allowed_tool_entries(tool_choice: Mapping[str, object]) -> Sequence[object] | None:
|
||||
if tool_choice.get("type") != "allowed_tools":
|
||||
return None
|
||||
|
|
@ -682,19 +740,19 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
response["stop_reason"] = "end_turn" # rebind-ok: dropping every tool_use ends the turn
|
||||
|
||||
def _get_request_tool_targets(self, tool: object, call_type: CallTypesLiteral) -> tuple[_ToolTarget, ...]:
|
||||
members: Final = _namespace_members(tool)
|
||||
if members:
|
||||
namespace: Final = _non_empty_string(self._get_mapping_value(tool, "name"))
|
||||
member_targets: Final = (_namespace_member_target(namespace, member) for member in members)
|
||||
return tuple(target for target in member_targets if target is not None)
|
||||
group: Final = _member_group(tool)
|
||||
if group is not None:
|
||||
return tuple(target for target in map(group.target_of, group.members) if target is not None)
|
||||
tool_type: Final = self._get_mapping_value(tool, "type")
|
||||
normalized_type: Final = (
|
||||
tool_type if call_type in _RESPONSES_CALL_TYPES or tool_type not in (None, "custom") else "function"
|
||||
)
|
||||
reads_as_function: Final = tool_type is None or (tool_type == "custom" and call_type in _ANTHROPIC_CALL_TYPES)
|
||||
normalized_type: Final = "function" if reads_as_function else tool_type
|
||||
names: Final = _request_tool_names(tool)
|
||||
if names:
|
||||
return tuple((name, normalized_type) for name in names)
|
||||
return ((None, normalized_type),) if isinstance(tool_type, str) else ()
|
||||
if isinstance(tool_type, str):
|
||||
return ((None, tool_type),)
|
||||
sole_key: Final = _sole_key(tool)
|
||||
return ((None, sole_key),) if sole_key is not None else ()
|
||||
|
||||
def _get_legacy_function_name(self, function: object) -> str | None:
|
||||
return self._get_mapping_value(function, "name")
|
||||
|
|
@ -704,10 +762,18 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
return None
|
||||
if isinstance(tool_choice, str):
|
||||
return tool_choice
|
||||
if self._get_mapping_value(tool_choice, "type") not in ("tool", "function", "custom"):
|
||||
choice: Final = _object_mapping(tool_choice)
|
||||
if choice is None:
|
||||
return None
|
||||
function_name: Final = self._get_mapping_value(self._get_mapping_value(tool_choice, "function"), "name")
|
||||
return function_name or self._get_mapping_value(tool_choice, "name")
|
||||
choice_type: Final = choice.get("type")
|
||||
server_field: Final = _HOSTED_SERVER_NAME_FIELDS.get(str(choice_type))
|
||||
if server_field is not None:
|
||||
return _non_empty_string(choice.get(server_field))
|
||||
if choice_type not in ("tool", "function", "custom"):
|
||||
return None
|
||||
nested: Final = _object_mapping(choice.get(str(choice_type)))
|
||||
nested_name: Final = _non_empty_string(nested.get("name")) if nested is not None else None
|
||||
return nested_name or _non_empty_string(choice.get("name"))
|
||||
|
||||
def _get_named_function_call(self, data: dict) -> str | None:
|
||||
function_call: Final = data.get("function_call")
|
||||
|
|
@ -728,15 +794,18 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
if function_name is not None:
|
||||
request_tools.append((function_name, "function"))
|
||||
|
||||
listed_names: Final = frozenset(name for name, _ in request_tools)
|
||||
for forced_tool_name in (
|
||||
self._get_named_tool_choice(_REQUEST_DATA.validate_python(data.get("tool_choice"))),
|
||||
self._get_named_function_call(data),
|
||||
):
|
||||
if forced_tool_name is not None:
|
||||
if forced_tool_name is not None and forced_tool_name not in listed_names:
|
||||
request_tools.append((forced_tool_name, "function"))
|
||||
|
||||
replayed: Final = map(_function_tool_name, replayed_tool_search_tools(_REQUEST_DATA.validate_python(data)))
|
||||
request_tools.extend((name, "function") for name in replayed if name is not None)
|
||||
for loaded_tool in _loaded_input_tools(data):
|
||||
request_tools.extend(self._get_request_tool_targets(loaded_tool, call_type))
|
||||
|
||||
return request_tools
|
||||
|
||||
|
|
@ -751,29 +820,51 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
denied_names: frozenset[str],
|
||||
call_type: CallTypesLiteral,
|
||||
) -> tuple[object, ...]:
|
||||
members: Final = _namespace_members(tool)
|
||||
group: Final = _member_group(tool)
|
||||
fields: Final = _object_mapping(tool)
|
||||
if not members or fields is None:
|
||||
if group is None or fields is None:
|
||||
targets: Final = self._get_request_tool_targets(tool, call_type)
|
||||
denied: Final = any(_is_denied_target(target, denied_targets, denied_names) for target in targets)
|
||||
return () if denied else (tool,)
|
||||
namespace: Final = _non_empty_string(fields.get("name"))
|
||||
kept_members: Final = [
|
||||
member
|
||||
for member in members
|
||||
if not _is_denied_target(
|
||||
_namespace_member_target(namespace, member) or (None, None), denied_targets, denied_names
|
||||
)
|
||||
for member in group.members
|
||||
if not _is_denied_target(group.target_of(member) or (None, None), denied_targets, denied_names)
|
||||
]
|
||||
if not kept_members:
|
||||
return ()
|
||||
return (tool,) if len(kept_members) == len(members) else ({**fields, "tools": kept_members},)
|
||||
return (tool,) if len(kept_members) == len(group.members) else ({**fields, group.key: kept_members},)
|
||||
|
||||
def _rewritten_tool_choice(self, tool_choice: object, removed_names: frozenset[str], tools_remain: bool) -> object:
|
||||
def _with_kept_loaded_tools(
|
||||
self,
|
||||
item: object,
|
||||
denied_targets: frozenset[_ToolTarget],
|
||||
denied_names: frozenset[str],
|
||||
call_type: CallTypesLiteral,
|
||||
) -> object:
|
||||
fields: Final = _object_mapping(item)
|
||||
loaded: Final = _tool_search_output_tools(item)
|
||||
if fields is None or loaded is None:
|
||||
return item
|
||||
kept: Final = chain.from_iterable(
|
||||
self._kept_tools(tool, denied_targets, denied_names, call_type) for tool in loaded
|
||||
)
|
||||
return {**fields, "tools": list(kept)}
|
||||
|
||||
def _tool_types_in(self, tools: Sequence[object], call_type: CallTypesLiteral) -> frozenset[str]:
|
||||
targets: Final = chain.from_iterable(self._get_request_tool_targets(tool, call_type) for tool in tools)
|
||||
return frozenset(tool_type for _, tool_type in targets if tool_type is not None)
|
||||
|
||||
def _rewritten_tool_choice(
|
||||
self, tool_choice: object, removed_names: frozenset[str], removed_types: frozenset[str], tools_remain: bool
|
||||
) -> object:
|
||||
named: Final = self._get_named_tool_choice(tool_choice)
|
||||
if named is not None and named in removed_names:
|
||||
return _none_tool_choice(tool_choice)
|
||||
choice: Final = _object_mapping(tool_choice)
|
||||
hosted_type: Final = choice.get("type") if choice is not None else None
|
||||
if hosted_type not in _NAMED_CHOICE_TYPES and hosted_type in removed_types:
|
||||
return _none_tool_choice(tool_choice)
|
||||
allowed: Final = _allowed_tool_entries(choice) if choice is not None else None
|
||||
if choice is not None and allowed is not None:
|
||||
kept: Final = [
|
||||
|
|
@ -824,12 +915,23 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
if tools is not None and kept_tools is not None
|
||||
else frozenset()
|
||||
)
|
||||
removed_types: Final = (
|
||||
self._tool_types_in(tools, call_type) - self._tool_types_in(kept_tools, call_type)
|
||||
if tools is not None and kept_tools is not None
|
||||
else frozenset()
|
||||
)
|
||||
tool_choice: Final = _REQUEST_DATA.validate_python(data.get("tool_choice"))
|
||||
if tool_choice is not None:
|
||||
data["tool_choice"] = self._rewritten_tool_choice(
|
||||
tool_choice, removed_names, bool(kept_tools) or bool(kept_functions)
|
||||
tool_choice, removed_names, removed_types, bool(kept_tools) or bool(kept_functions)
|
||||
)
|
||||
|
||||
input_items: Final = _input_items(data)
|
||||
if any(_tool_search_output_tools(item) is not None for item in input_items):
|
||||
data["input"] = [
|
||||
self._with_kept_loaded_tools(item, denied_targets, denied_names, call_type) for item in input_items
|
||||
]
|
||||
|
||||
named_function_call: Final = self._get_named_function_call(data)
|
||||
if named_function_call in denied_names:
|
||||
data["function_call"] = "none"
|
||||
|
|
|
|||
|
|
@ -1424,13 +1424,43 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
{"id": "deny_mcp", "tool_type": r"^mcp$", "decision": "deny"},
|
||||
{"type": "mcp", "server_label": "weather", "server_url": "https://mcp.example.com"},
|
||||
"responses",
|
||||
"Tool 'mcp' denied by rule 'deny_mcp'",
|
||||
"Tool 'weather' denied by rule 'deny_mcp'",
|
||||
),
|
||||
(
|
||||
{"id": "allow_bash", "tool_name": r"^Bash$", "decision": "allow"},
|
||||
{"type": "mcp_toolset", "mcp_server_name": "weather"},
|
||||
"anthropic_messages",
|
||||
"Tool 'mcp_toolset' denied by default action",
|
||||
"Tool 'weather' denied by default action",
|
||||
),
|
||||
(
|
||||
{"id": "deny_weather", "tool_name": r"^weather$", "decision": "deny"},
|
||||
{"type": "mcp", "server_label": "weather", "server_url": "https://mcp.example.com"},
|
||||
"responses",
|
||||
"Tool 'weather' denied by rule 'deny_weather'",
|
||||
),
|
||||
(
|
||||
{"id": "deny_lookup", "tool_name": r"^mcp__weather__lookup$", "decision": "deny"},
|
||||
{"type": "namespace", "name": "mcp__weather", "tools": [{"type": "custom", "name": "lookup"}]},
|
||||
"responses",
|
||||
"Tool 'mcp__weather__lookup' denied by rule 'deny_lookup'",
|
||||
),
|
||||
(
|
||||
{"id": "deny_lookup", "tool_name": r"^mcp__weather__lookup$", "decision": "deny"},
|
||||
{"functionDeclarations": [{"name": "mcp__weather__lookup"}, {"name": "get_weather"}]},
|
||||
"agenerate_content",
|
||||
"Tool 'mcp__weather__lookup' denied by rule 'deny_lookup'",
|
||||
),
|
||||
(
|
||||
{"id": "allow_bash", "tool_name": r"^Bash$", "decision": "allow"},
|
||||
{"googleSearch": {}},
|
||||
"acompletion",
|
||||
"Tool 'googleSearch' denied by default action",
|
||||
),
|
||||
(
|
||||
{"id": "deny_custom", "tool_type": r"^custom$", "decision": "deny"},
|
||||
{"type": "custom", "custom": {"name": "grammar_tool"}},
|
||||
"acompletion",
|
||||
"Tool 'grammar_tool' denied by rule 'deny_custom'",
|
||||
),
|
||||
(
|
||||
{"id": "deny_read", "tool_name": r"^Read$", "decision": "deny"},
|
||||
|
|
@ -1439,7 +1469,16 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
"Tool 'Read' denied by rule 'deny_read'",
|
||||
),
|
||||
],
|
||||
ids=["responses_mcp_by_type", "anthropic_mcp_toolset_by_default", "chat_custom_nested_name"],
|
||||
ids=[
|
||||
"responses_mcp_by_type",
|
||||
"anthropic_mcp_toolset_by_server_name",
|
||||
"responses_mcp_by_server_label",
|
||||
"responses_namespace_custom_member",
|
||||
"gemini_function_declarations",
|
||||
"gemini_type_only_tool_by_its_key",
|
||||
"chat_custom_keeps_its_type",
|
||||
"chat_custom_nested_name",
|
||||
],
|
||||
)
|
||||
async def test_pre_call_checks_tools_that_carry_no_top_level_name(
|
||||
self, rule: dict[str, str], tool: dict[str, object], call_type: CallTypesLiteral, detection_message: str
|
||||
|
|
@ -1453,14 +1492,77 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
assert excinfo.value.detail["detection_message"] == detection_message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_passes_a_tool_no_rule_could_ever_match(self) -> None:
|
||||
guardrail = self._pre_call(self.rules, "block", default_action="deny")
|
||||
async def test_pre_call_lets_a_type_rule_allow_a_gemini_tool_by_its_key(self) -> None:
|
||||
guardrail = self._pre_call(
|
||||
[{"id": "allow_search", "tool_type": r"^googleSearch$", "decision": "allow"}], "block", default_action="deny"
|
||||
)
|
||||
data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [{"googleSearch": {}}]}
|
||||
|
||||
result = await self._run_pre_call(guardrail, data, "acompletion")
|
||||
|
||||
assert result["tools"] == [{"googleSearch": {}}]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_blocks_a_denied_tool_loaded_by_responses_tool_search(self) -> None:
|
||||
guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "block")
|
||||
loaded = {"type": "tool_search_output", "tools": [{"type": "function", "name": "mcp__weather__lookup"}]}
|
||||
data = {"model": "gpt-5", "input": [loaded], "tools": []}
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await self._run_pre_call(guardrail, data, "responses")
|
||||
|
||||
assert excinfo.value.detail["detection_message"] == "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_rewrite_keeps_the_allowed_members_of_grouped_and_loaded_tools(self) -> None:
|
||||
guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite")
|
||||
get_weather = {"type": "function", "name": "get_weather"}
|
||||
lookup = {"type": "function", "name": "mcp__weather__lookup"}
|
||||
data = {
|
||||
"model": "m",
|
||||
"input": [{"role": "user", "content": "hi"}, {"type": "tool_search_output", "tools": [get_weather, lookup]}],
|
||||
"tools": [{"functionDeclarations": [{"name": "get_weather"}, {"name": "mcp__weather__lookup"}]}],
|
||||
}
|
||||
|
||||
result = await self._run_pre_call(guardrail, data, "responses")
|
||||
|
||||
assert result["tools"] == [{"functionDeclarations": [{"name": "get_weather"}]}]
|
||||
assert result["input"] == [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"type": "tool_search_output", "tools": [get_weather]},
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("rule", "tools", "tool_choice"),
|
||||
[
|
||||
(
|
||||
{"id": "deny_search", "tool_type": r"^web_search$", "decision": "deny"},
|
||||
[{"type": "web_search"}, {"type": "function", "name": "get_weather"}],
|
||||
{"type": "web_search"},
|
||||
),
|
||||
(
|
||||
{"id": "deny_weather", "tool_name": r"^weather$", "decision": "deny"},
|
||||
[
|
||||
{"type": "mcp", "server_label": "weather", "server_url": "https://a.example.com"},
|
||||
{"type": "mcp", "server_label": "files", "server_url": "https://b.example.com"},
|
||||
],
|
||||
{"type": "mcp", "server_label": "weather"},
|
||||
),
|
||||
],
|
||||
ids=["hosted_tool_type", "hosted_mcp_server"],
|
||||
)
|
||||
async def test_pre_call_rewrite_relaxes_a_hosted_tool_choice_it_removed(
|
||||
self, rule: dict[str, str], tools: list[dict[str, str]], tool_choice: dict[str, str]
|
||||
) -> None:
|
||||
guardrail = self._pre_call([rule], "rewrite")
|
||||
data = {"model": "gpt-5", "input": "hi", "tools": tools, "tool_choice": tool_choice}
|
||||
|
||||
result = await self._run_pre_call(guardrail, data, "responses")
|
||||
|
||||
assert result["tools"] == tools[1:]
|
||||
assert result["tool_choice"] == "none"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "tool_choice", "call_type", "expected_tool_choice"),
|
||||
|
|
@ -1514,6 +1616,12 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
"anthropic_messages",
|
||||
{"type": "any"},
|
||||
),
|
||||
(
|
||||
[{"type": "custom", "custom": {"name": "Bash"}}, {"type": "custom", "custom": {"name": "Read"}}],
|
||||
{"type": "custom", "custom": {"name": "Read"}},
|
||||
"acompletion",
|
||||
"none",
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"responses_custom_choice",
|
||||
|
|
@ -1522,6 +1630,7 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
"chat_allowed_tools_all_denied",
|
||||
"anthropic_any_with_no_tools_left",
|
||||
"anthropic_any_with_a_tool_left",
|
||||
"chat_custom_choice",
|
||||
],
|
||||
)
|
||||
async def test_pre_call_rewrite_leaves_no_tool_choice_that_needs_a_removed_tool(
|
||||
|
|
@ -1543,6 +1652,37 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
|
||||
assert result["tool_choice"] == expected_tool_choice
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tool", "tool_choice", "call_type", "allowed_tool_type"),
|
||||
[
|
||||
({"type": "custom", "name": "apply_patch"}, {"type": "custom", "name": "apply_patch"}, "responses", r"^custom$"),
|
||||
(
|
||||
{"type": "web_search_20250305", "name": "web_search"},
|
||||
{"type": "tool", "name": "web_search"},
|
||||
"anthropic_messages",
|
||||
r"^web_search_\d+$",
|
||||
),
|
||||
],
|
||||
ids=["responses_custom_tool", "anthropic_server_tool"],
|
||||
)
|
||||
async def test_pre_call_checks_a_forced_choice_with_its_tool_type(
|
||||
self,
|
||||
tool: dict[str, str],
|
||||
tool_choice: dict[str, str],
|
||||
call_type: CallTypesLiteral,
|
||||
allowed_tool_type: str,
|
||||
) -> None:
|
||||
guardrail = self._pre_call(
|
||||
[{"id": "allow_type", "tool_type": allowed_tool_type, "decision": "allow"}], "block", default_action="deny"
|
||||
)
|
||||
data = {"model": "m", "input": "hi", "tools": [tool], "tool_choice": tool_choice}
|
||||
|
||||
result = await self._run_pre_call(guardrail, data, call_type)
|
||||
|
||||
assert result["tools"] == [tool]
|
||||
assert result["tool_choice"] == tool_choice
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_rewrite_relaxes_a_choice_for_a_tool_a_type_rule_removed(self) -> None:
|
||||
guardrail = self._pre_call([{"id": "deny_custom", "tool_type": r"^custom$", "decision": "deny"}], "rewrite")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue