This commit is contained in:
Meryem Sakin 2026-10-29 01:55:19 +00:00 • committed by GitHub
commit 0a56e8a7ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 739 additions and 89 deletions

View file

@ -1,13 +1,13 @@
import json
import re
from collections.abc import AsyncGenerator, AsyncIterable, Mapping, Sequence
from typing import Any, Final, Literal, TypedDict
from itertools import chain
from typing import Any, Final, Literal, TypeAlias, TypedDict
from fastapi import HTTPException
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, Required
from litellm import ChatCompletionToolParam
from litellm._logging import verbose_proxy_logger
from litellm.caching.dual_cache import DualCache
from litellm.completion_extras.litellm_responses_transformation.transformation import (
@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.base_llm.guardrail_translation.utils import anthropic_tool_names
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
@ -45,8 +46,11 @@ from litellm.types.utils import (
)
GUARDRAIL_NAME: Final = "tool_permission"
_RESPONSES_CALL_TYPES: Final = frozenset({"responses", "aresponses", "_aresponses_websocket"})
_DECODED_ARGUMENTS: Final = TypeAdapter(object)
_REQUEST_DATA: Final = TypeAdapter(object)
_OPTIONAL_REQUEST_LIST: Final = TypeAdapter(list[object] | None)
_ToolTarget: TypeAlias = tuple[str | None, str | None]
def _object_mapping(value: object) -> Mapping[str, object] | None:
@ -65,6 +69,67 @@ def _function_tool_name(tool: object) -> str | None:
return name if isinstance(name, str) else None
def _non_empty_string(value: object) -> str | None:
return value if isinstance(value, str) and value else None
def _request_tool_names(tool: object) -> tuple[str, ...]:
fields: Final = _object_mapping(tool)
custom: Final = (
_object_mapping(fields.get("custom")) if fields is not None and fields.get("type") == "custom" else None
)
custom_name: Final = _non_empty_string(custom.get("name")) if custom is not None else None
return anthropic_tool_names(tool) + ((custom_name,) if custom_name is not None else ())
def _namespace_members(tool: object) -> Sequence[object]:
fields: Final = _object_mapping(tool)
members: Final = (
_object_list(fields.get("tools")) if fields is not None and fields.get("type") == "namespace" else None
)
return members or ()
def _namespace_member_target(namespace: str | None, member: object) -> _ToolTarget | None:
fields: Final = _object_mapping(member)
if fields is None:
return None
member_type: Final = _non_empty_string(fields.get("type"))
name: Final = _non_empty_string(fields.get("name"))
qualified: Final = name is not None and namespace is not None and member_type != "custom"
return (f"{namespace}__{name}" if qualified else name, member_type)
def _allowed_tool_entries(tool_choice: Mapping[str, object]) -> Sequence[object] | None:
if tool_choice.get("type") != "allowed_tools":
return None
nested: Final = _object_mapping(tool_choice.get("allowed_tools"))
return _object_list((nested if nested is not None else tool_choice).get("tools"))
def _with_allowed_tool_entries(tool_choice: Mapping[str, object], entries: list[object]) -> dict[str, object]:
nested: Final = _object_mapping(tool_choice.get("allowed_tools"))
if nested is None:
return {**tool_choice, "tools": entries}
return {**tool_choice, "allowed_tools": {**nested, "tools": entries}}
def _is_denied_target(
target: _ToolTarget, denied_targets: frozenset[_ToolTarget], denied_names: frozenset[str]
) -> bool:
return target in denied_targets or target[0] in denied_names
def _none_tool_choice(tool_choice: object) -> object:
choice: Final = _object_mapping(tool_choice)
return {"type": "none"} if choice is not None and choice.get("type") in ("tool", "any") else "none"
def _requires_a_tool(tool_choice: object) -> bool:
choice: Final = _object_mapping(tool_choice)
return tool_choice == "required" or (choice is not None and choice.get("type") == "any")
class _ToolPermissionRuleFields(TypedDict, total=False):
"""The config-file shape a :class:`ToolPermissionRule` is built from."""
@ -616,27 +681,33 @@ class ToolPermissionGuardrail(CustomGuardrail):
if not any(_is_tool_use_block(block) for block in kept_blocks):
response["stop_reason"] = "end_turn" # rebind-ok: dropping every tool_use ends the turn
def _get_request_tool_name(self, tool: object) -> tuple[str | None, str | None]:
def _get_request_tool_targets(self, tool: object, call_type: CallTypesLiteral) -> tuple[_ToolTarget, ...]:
members: Final = _namespace_members(tool)
if members:
namespace: Final = _non_empty_string(self._get_mapping_value(tool, "name"))
member_targets: Final = (_namespace_member_target(namespace, member) for member in members)
return tuple(target for target in member_targets if target is not None)
tool_type: Final = self._get_mapping_value(tool, "type")
if tool_type != "function":
return None, tool_type
function: Final = self._get_mapping_value(tool, "function")
tool_name: Final = self._get_mapping_value(function, "name")
return tool_name, tool_type
normalized_type: Final = (
tool_type if call_type in _RESPONSES_CALL_TYPES or tool_type not in (None, "custom") else "function"
)
names: Final = _request_tool_names(tool)
if names:
return tuple((name, normalized_type) for name in names)
return ((None, normalized_type),) if isinstance(tool_type, str) else ()
def _get_legacy_function_name(self, function: object) -> str | None:
return self._get_mapping_value(function, "name")
def _get_named_tool_choice(self, data: dict) -> str | None:
tool_choice: Final = data.get("tool_choice")
def _get_named_tool_choice(self, tool_choice: object) -> str | None:
if not tool_choice or tool_choice in ("auto", "none", "required"):
return None
if isinstance(tool_choice, str):
return tool_choice
if self._get_mapping_value(tool_choice, "type") != "function":
if self._get_mapping_value(tool_choice, "type") not in ("tool", "function", "custom"):
return None
return self._get_mapping_value(self._get_mapping_value(tool_choice, "function"), "name")
function_name: Final = self._get_mapping_value(self._get_mapping_value(tool_choice, "function"), "name")
return function_name or self._get_mapping_value(tool_choice, "name")
def _get_named_function_call(self, data: dict) -> str | None:
function_call: Final = data.get("function_call")
@ -646,13 +717,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
return function_call
return self._get_mapping_value(function_call, "name")
def _collect_request_tools(self, data: dict) -> list[tuple[str, str | None]]:
request_tools: Final[list[tuple[str, str | None]]] = []
def _collect_request_tools(self, data: dict, call_type: CallTypesLiteral) -> list[_ToolTarget]:
request_tools: Final[list[_ToolTarget]] = []
for tool in data.get("tools") or []:
tool_name, tool_type = self._get_request_tool_name(tool)
if tool_name is not None:
request_tools.append((tool_name, tool_type))
request_tools.extend(self._get_request_tool_targets(tool, call_type))
for function in data.get("functions") or []:
function_name = self._get_legacy_function_name(function)
@ -660,7 +729,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
request_tools.append((function_name, "function"))
for forced_tool_name in (
self._get_named_tool_choice(data),
self._get_named_tool_choice(_REQUEST_DATA.validate_python(data.get("tool_choice"))),
self._get_named_function_call(data),
):
if forced_tool_name is not None:
@ -671,55 +740,101 @@ class ToolPermissionGuardrail(CustomGuardrail):
return request_tools
def _tool_names_in(self, tools: Sequence[object], call_type: CallTypesLiteral) -> frozenset[str]:
targets: Final = chain.from_iterable(self._get_request_tool_targets(tool, call_type) for tool in tools)
return frozenset(name for name, _ in targets if name is not None)
def _kept_tools(
self,
tool: object,
denied_targets: frozenset[_ToolTarget],
denied_names: frozenset[str],
call_type: CallTypesLiteral,
) -> tuple[object, ...]:
members: Final = _namespace_members(tool)
fields: Final = _object_mapping(tool)
if not members or fields is None:
targets: Final = self._get_request_tool_targets(tool, call_type)
denied: Final = any(_is_denied_target(target, denied_targets, denied_names) for target in targets)
return () if denied else (tool,)
namespace: Final = _non_empty_string(fields.get("name"))
kept_members: Final = [
member
for member in members
if not _is_denied_target(
_namespace_member_target(namespace, member) or (None, None), denied_targets, denied_names
)
]
if not kept_members:
return ()
return (tool,) if len(kept_members) == len(members) else ({**fields, "tools": kept_members},)
def _rewritten_tool_choice(self, tool_choice: object, removed_names: frozenset[str], tools_remain: bool) -> object:
named: Final = self._get_named_tool_choice(tool_choice)
if named is not None and named in removed_names:
return _none_tool_choice(tool_choice)
choice: Final = _object_mapping(tool_choice)
allowed: Final = _allowed_tool_entries(choice) if choice is not None else None
if choice is not None and allowed is not None:
kept: Final = [
entry for entry in allowed if not any(name in removed_names for name in _request_tool_names(entry))
]
return _with_allowed_tool_entries(choice, kept) if kept and tools_remain else _none_tool_choice(choice)
if not tools_remain and (named is not None or _requires_a_tool(tool_choice)):
return _none_tool_choice(tool_choice)
return tool_choice
def _modify_request_with_permission_errors(
self,
data: dict,
denied_tool_names: list[str],
denied_targets: frozenset[_ToolTarget],
call_type: CallTypesLiteral,
):
"""
Modify the request to replace denied tool_calls blocks with error results
Args:
data: The model request to modify
denied_tools: List of (tool_use, error) tuples for denied tools
Remove denied tools from the request, and relax any tool_choice that would still require one
"""
if not denied_tool_names:
if not denied_targets:
return data
verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tool_names))
verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_targets))
# Create a mapping of tool_use_id to error result
error_tool_names: Final = set()
for tool_use in denied_tool_names:
error_tool_names.add(tool_use)
denied_names: Final = frozenset(name for name, _ in denied_targets if name is not None)
tools: Final = _OPTIONAL_REQUEST_LIST.validate_python(data.get("tools"))
kept_tools: Final = (
None
if tools is None
else list(
chain.from_iterable(self._kept_tools(tool, denied_targets, denied_names, call_type) for tool in tools)
)
)
if kept_tools is not None:
data["tools"] = kept_tools
tools: Final[list[ChatCompletionToolParam] | None] = data.get("tools")
if tools is not None:
new_tools: Final = []
for tool in tools:
tool_name, tool_type = self._get_request_tool_name(tool)
if tool_type == "function" and tool_name in error_tool_names:
continue
new_tools.append(tool)
data["tools"] = new_tools
functions: Final = _OPTIONAL_REQUEST_LIST.validate_python(data.get("functions"))
kept_functions: Final = (
None
if functions is None
else [function for function in functions if self._get_legacy_function_name(function) not in denied_names]
)
if kept_functions is not None:
data["functions"] = kept_functions
functions: Final = data.get("functions")
if functions is not None:
data["functions"] = [
function for function in functions if self._get_legacy_function_name(function) not in error_tool_names
]
named_tool_choice: Final = self._get_named_tool_choice(data)
if named_tool_choice in error_tool_names:
data["tool_choice"] = "none"
removed_names: Final = denied_names | (
self._tool_names_in(tools, call_type) - self._tool_names_in(kept_tools, call_type)
if tools is not None and kept_tools is not None
else frozenset()
)
tool_choice: Final = _REQUEST_DATA.validate_python(data.get("tool_choice"))
if tool_choice is not None:
data["tool_choice"] = self._rewritten_tool_choice(
tool_choice, removed_names, bool(kept_tools) or bool(kept_functions)
)
named_function_call: Final = self._get_named_function_call(data)
if named_function_call in error_tool_names:
if named_function_call in denied_names:
data["function_call"] = "none"
messages: Final = without_tool_search_items_loading(
_REQUEST_DATA.validate_python(data), frozenset(denied_tool_names)
)
messages: Final = without_tool_search_items_loading(_REQUEST_DATA.validate_python(data), denied_names)
if messages is not None:
data["messages"] = messages # rebind-ok: rewrite mode edits the request in place
@ -827,32 +942,32 @@ class ToolPermissionGuardrail(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return data
new_tools: Final = self._collect_request_tools(data)
new_tools: Final = self._collect_request_tools(data, call_type)
if not new_tools:
verbose_proxy_logger.debug(
"Tool Permission Guardrail: not running guardrail. No tools or functions in data"
)
return data
# Check permissions for each tool
denied_tool_names: Final = []
for tool_name, tool_type in new_tools:
is_allowed, _, message = self._check_tool_permission(tool_name, tool_type)
decisions: Final = tuple((target, self._check_tool_permission(*target)) for target in new_tools)
denied: Final = tuple(
(target, message)
for target, (is_allowed, _, message) in decisions
if not is_allowed and message is not None
)
for _, message in denied:
verbose_proxy_logger.info("Tool Permission Guardrail: %s", message)
if denied and self.on_disallowed_action == "block":
raise HTTPException(
status_code=400,
detail={
"error": "Violated guardrail policy",
"detection_message": denied[0][1],
},
)
if not is_allowed and message is not None:
verbose_proxy_logger.info("Tool Permission Guardrail: %s", message)
if self.on_disallowed_action == "block":
raise HTTPException(
status_code=400,
detail={
"error": "Violated guardrail policy",
"detection_message": message,
},
)
denied_tool_names.append(tool_name)
if denied_tool_names:
data = self._modify_request_with_permission_errors(data, denied_tool_names)
if denied:
self._modify_request_with_permission_errors(data, frozenset(target for target, _ in denied), call_type)
verbose_proxy_logger.debug("Tool Permission Guardrail Pre-Call Hook: All tools allowed")

View file

@ -2048,7 +2048,7 @@ _WS_FRAME: Final = TypeAdapter(dict[str, object])
_GUARDRAIL_INFORMATION: Final = TypeAdapter(list[dict[str, object]])
_PII_TOKENS: Final = TypeAdapter(dict[str, str])
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
_GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools")
_GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools", "tool_choice")
_CLIENT_METADATA_KEYS: Final = ("metadata", "litellm_metadata")
# the guardrail list is merged from key, team and the admission frame; a later frame cannot shrink it
_CLIENT_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config", "policies")
@ -2092,7 +2092,7 @@ def _responses_ws_frame_guard(
nested: Final = frame_obj.get("response")
if isinstance(nested, dict) and any(key in frame_obj for key in _GUARDED_FRAME_FIELDS):
raise ValueError(
"response.create carries input, instructions or tools both at the top level and under response"
"response.create carries input, instructions, tools or tool_choice both at the top level and under response"
)
body: Final = _WS_FRAME.validate_python(nested) if isinstance(nested, dict) else frame_obj
frame_data: Final = {

View file

@ -25,8 +25,8 @@
- {id: guardrail.ibm_guardrails.post_call.blocks, module: guardrail, tier: P1, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/ibm_guardrails/ibm_detector.py", rationale: "Output policy validation"}
- {id: guardrail.semantic_guard.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/semantic_guard", rationale: "Semantic policy compliance"}
- {id: guardrail.block_code_execution.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/block_code_execution", rationale: "Code-injection prevention"}
- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"}
- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"}
- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"}
- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"}
- {id: guardrail.microsoft_purview.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/microsoft_purview/purview_dlp.py", rationale: "DLP sensitive-data disclosure"}
- {id: guardrail.headroom.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/headroom/headroom.py", rationale: "Anomaly detection threshold"}
- {id: guardrail.generic_guardrail_api.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py", rationale: "Vendor-agnostic custom API"}

View file

@ -19,10 +19,9 @@ verdict comes from the proxy itself.
from __future__ import annotations
from typing import Final
from typing import Final, Literal
import pytest
from e2e_config import unique_marker
from e2e_http import StreamingResponse, UnknownApiError
from e2e_metadata import Capability, Domain, Mode, Provider, Subject, meta
@ -31,14 +30,29 @@ from guardrails_client import (
ToolPermissionParamsBody,
ToolPermissionRuleBody,
poll_until_blocked,
poll_until_blocked_stream,
poll_until_guardrail_applied,
)
from lifecycle import ResourceManager
from models import ChatResponse, ChatTool, ChatToolFunction
from models import (
AnthropicCustomTool,
AnthropicMessagesBody,
AnthropicMessagesResponse,
AnthropicToolChoice,
ChatMessage,
ChatResponse,
ChatTool,
ChatToolFunction,
JsonSchemaProperty,
ResponsesFunctionTool,
ResponsesToolBody,
ResponsesToolOutput,
ToolInputSchema,
)
pytestmark = pytest.mark.e2e
pytestmark: Final = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
MODEL: Final = "gemini-2.5-flash"
#: The one tool the guardrail permits, and one it does not. Both are declared by
#: the caller in the request body; the guardrail reads them there.
@ -63,11 +77,15 @@ DENIED_TOOL: Final = ChatTool(
TOOL_PROMPT: Final = "What is the weather in Paris right now?"
type FlatToolEndpoint = Literal["messages", "responses"]
ALLOWED_TOOL_SCHEMA: Final = ToolInputSchema(properties={"city": JsonSchemaProperty(type="string")}, required=["city"])
DENIED_TOOL_SCHEMA: Final = ToolInputSchema()
def _register_tool_permission(client: GuardrailsClient, resources: ResourceManager, *, name: str) -> None:
"""Allow-list exactly one tool: everything else falls to `default_action=deny`
and, with `on_disallowed_action=block`, is rejected outright."""
guardrail_id = client.register(
guardrail_id: Final = client.register(
name,
ToolPermissionParamsBody(
mode="pre_call",
@ -100,6 +118,57 @@ def _tool_call_names(response: ChatResponse) -> tuple[str, ...]:
)
def _send_flat_tool(
client: GuardrailsClient,
key: str,
endpoint: FlatToolEndpoint,
*,
guardrail: str,
tool: ChatTool,
schema: ToolInputSchema,
require_tool: bool = False,
) -> StreamingResponse:
name: Final = tool.function.name
match endpoint:
case "messages":
return client.proxy.transport.send(
"/v1/messages",
headers=client.proxy.transport.bearer(key),
json=AnthropicMessagesBody(
model=MODEL,
messages=[ChatMessage(role="user", content=TOOL_PROMPT)],
max_tokens=128,
tools=[
AnthropicCustomTool(name=name, description=tool.function.description or "", input_schema=schema)
],
tool_choice=AnthropicToolChoice(type="any") if require_tool else None,
guardrails=[guardrail],
),
)
case "responses":
return client.proxy.transport.send(
"/v1/responses",
headers=client.proxy.transport.bearer(key),
json=ResponsesToolBody(
model=MODEL,
input=TOOL_PROMPT,
tools=[ResponsesFunctionTool(name=name, description=tool.function.description, parameters=schema)],
tool_choice="required" if require_tool else None,
guardrails=[guardrail],
),
)
def _flat_tool_call_names(endpoint: FlatToolEndpoint, body: str) -> tuple[str, ...]:
match endpoint:
case "messages":
content: Final = AnthropicMessagesResponse.model_validate_json(body).content or []
return tuple(block.name for block in content if block.type == "tool_use" and block.name)
case "responses":
output: Final = ResponsesToolOutput.model_validate_json(body).output
return tuple(item.name for item in output if item.type == "function_call" and item.name)
class TestToolPermissionPreCall:
@pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["chat_completions"])
@meta(
@ -118,10 +187,10 @@ class TestToolPermissionPreCall:
rejected with a 400 that names the denied tool. An unauthorized tool that
merely reaches the model is the whole failure mode this guardrail exists
to prevent, so a 200 here is a hard failure."""
name = f"e2e-toolperm-block-{unique_marker()}"
name: Final = f"e2e-toolperm-block-{unique_marker()}"
_register_tool_permission(client, resources, name=name)
result = poll_until_blocked(
result: Final = poll_until_blocked(
lambda: client.chat(
scoped_key,
MODEL,
@ -161,10 +230,10 @@ class TestToolPermissionPreCall:
and the model calls it. Without the header check a guardrail that never
attached would pass this test for the wrong reason, so the 200 alone is
not the contract."""
name = f"e2e-toolperm-allow-{unique_marker()}"
name: Final = f"e2e-toolperm-allow-{unique_marker()}"
_register_tool_permission(client, resources, name=name)
outcome = poll_until_guardrail_applied(
outcome: Final = poll_until_guardrail_applied(
lambda: client.chat_raw(
scoped_key,
MODEL,
@ -178,13 +247,117 @@ class TestToolPermissionPreCall:
)
assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}"
applied = _applied_guardrails(outcome)
applied: Final = _applied_guardrails(outcome)
assert name in applied, (
"the allowed call must carry x-litellm-applied-guardrails naming the guardrail; "
f"without it the 200 only proves the guardrail never ran. Got {applied!r}"
)
called = _tool_call_names(ChatResponse.model_validate_json(outcome.body))
called: Final = _tool_call_names(ChatResponse.model_validate_json(outcome.body))
assert called == (ALLOWED_TOOL.function.name,), (
f"the served call must carry one tool call for the permitted tool, got {called!r}: {outcome.body[:400]}"
)
@pytest.mark.parametrize(
"endpoint",
[
pytest.param(
"messages",
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["messages"]),
),
pytest.param(
"responses",
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["responses"]),
),
],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
providers=(Provider.GEMINI,),
models=(MODEL,),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_pre_call_blocks_flat_tool_outside_the_allow_list(
self,
client: GuardrailsClient,
resources: ResourceManager,
scoped_key: str,
endpoint: FlatToolEndpoint,
) -> None:
name: Final = f"e2e-toolperm-block-{endpoint}-{unique_marker()}"
_register_tool_permission(client, resources, name=name)
outcome: Final = poll_until_blocked_stream(
lambda: _send_flat_tool(
client, scoped_key, endpoint, guardrail=name, tool=DENIED_TOOL, schema=DENIED_TOOL_SCHEMA
)
)
assert outcome.status_code == 400, (
f"tool_permission let a tool outside the allow-list through /{endpoint}; "
f"got {outcome.status_code}: {outcome.body[:400]}"
)
assert DENIED_TOOL.function.name in outcome.body, (
f"the block must name the denied tool so the caller can fix the request; got: {outcome.body[:400]}"
)
assert "Violated guardrail policy" in outcome.body, (
f"the block must be a guardrail verdict, got: {outcome.body[:400]}"
)
@pytest.mark.parametrize(
"endpoint",
[
pytest.param(
"messages",
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["messages"]),
),
pytest.param(
"responses",
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["responses"]),
),
],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
providers=(Provider.GEMINI,),
models=(MODEL,),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_pre_call_allows_permitted_flat_tool(
self,
client: GuardrailsClient,
resources: ResourceManager,
scoped_key: str,
endpoint: FlatToolEndpoint,
) -> None:
name: Final = f"e2e-toolperm-allow-{endpoint}-{unique_marker()}"
_register_tool_permission(client, resources, name=name)
outcome: Final = poll_until_guardrail_applied(
lambda: _send_flat_tool(
client,
scoped_key,
endpoint,
guardrail=name,
tool=ALLOWED_TOOL,
schema=ALLOWED_TOOL_SCHEMA,
require_tool=True,
),
name,
)
assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}"
applied: Final = _applied_guardrails(outcome)
assert name in applied, (
f"the allowed call must carry x-litellm-applied-guardrails naming the guardrail; got {applied!r}"
)
called: Final = _flat_tool_call_names(endpoint, outcome.body)
assert called == (ALLOWED_TOOL.function.name,), (
f"the served call must carry one tool call for the permitted tool, got {called!r}: {outcome.body[:400]}"
)

View file

@ -655,6 +655,35 @@ class ResponsesStreamBody(BaseModel):
cache: dict[str, bool] | None = {"no-cache": True}
class ResponsesFunctionTool(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["function"] = "function"
name: str
description: str | None = None
parameters: ToolInputSchema
class ResponsesToolBody(BaseModel):
model_config = ConfigDict(frozen=True)
model: str
input: str
tools: list[ResponsesFunctionTool]
tool_choice: Literal["required"] | None = None
guardrails: list[str]
max_output_tokens: int = 128
class ResponsesOutputItem(BaseModel):
model_config = ConfigDict(frozen=True)
type: str
name: str | None = None
class ResponsesToolOutput(BaseModel):
model_config = ConfigDict(frozen=True)
output: list[ResponsesOutputItem]
class CountTokensBody(BaseModel):
"""POST /v1/messages/count_tokens body: the /v1/messages shape minus
max_tokens (the endpoint only counts the prompt)."""

View file

@ -5,7 +5,7 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics)
import json
import logging
import re
from typing import Final
from typing import Final, Literal
from unittest.mock import patch
import pytest
@ -25,6 +25,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
PermissionError,
)
from litellm.types.utils import (
CallTypesLiteral,
ChatCompletionMessageToolCall,
Choices,
ModelResponse,
@ -1250,6 +1251,313 @@ class TestToolPermissionGuardrailAnthropicMessages:
def _tool_use(self, name, tool_id="tu_1"):
return {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": "ls"}}
def _always_on_pre_call(self, on_disallowed_action: Literal["block", "rewrite"]) -> ToolPermissionGuardrail:
return ToolPermissionGuardrail(
guardrail_name=f"anthropic-pre-call-{on_disallowed_action}",
rules=self.rules,
default_action="deny",
on_disallowed_action=on_disallowed_action,
event_hook=GuardrailEventHooks.pre_call,
default_on=True,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"denied_tool",
[
{"name": "Read", "input_schema": {"type": "object", "properties": {}}},
{"type": "custom", "name": "Read", "input_schema": {"type": "object", "properties": {}}},
{"type": "function", "name": "Read", "parameters": {"type": "object", "properties": {}}},
],
ids=["anthropic", "anthropic_custom_type", "responses_api_flat_function"],
)
async def test_pre_call_blocks_denied_request_tool_in_flat_format(self, denied_tool: dict[str, object]) -> None:
data = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "tools": [denied_tool]}
with pytest.raises(HTTPException) as excinfo:
await self._always_on_pre_call("block").async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(),
cache=DualCache(default_in_memory_ttl=1),
data=data,
call_type="anthropic_messages",
)
assert excinfo.value.status_code == 400
assert excinfo.value.detail["detection_message"] == "Tool 'Read' denied by rule 'deny_read'"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("rule_tool_type", "call_type"),
[(r"^custom$", "responses"), (r"^function$", "anthropic_messages")],
ids=["responses_custom_stays_custom", "anthropic_custom_reads_as_function"],
)
async def test_pre_call_tool_type_rules_follow_the_request_format(
self, rule_tool_type: str, call_type: CallTypesLiteral
) -> None:
guardrail = ToolPermissionGuardrail(
guardrail_name=f"tool-type-{call_type}",
rules=[{"id": "deny_type", "tool_type": rule_tool_type, "decision": "deny"}],
default_action="allow",
on_disallowed_action="block",
event_hook=GuardrailEventHooks.pre_call,
default_on=True,
)
data = {
"model": "claude-sonnet-4-5",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"type": "custom", "name": "apply_patch", "input_schema": {"type": "object", "properties": {}}}],
}
with pytest.raises(HTTPException) as excinfo:
await guardrail.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(),
cache=DualCache(default_in_memory_ttl=1),
data=data,
call_type=call_type,
)
assert excinfo.value.status_code == 400
assert excinfo.value.detail["detection_message"] == "Tool 'apply_patch' denied by rule 'deny_type'"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool_shape", "tool_choice", "call_type", "expected_tool_choice"),
[
(
{"input_schema": {"type": "object", "properties": {}}},
{"type": "tool", "name": "Read"},
"anthropic_messages",
{"type": "none"},
),
(
{"type": "function", "parameters": {"type": "object", "properties": {}}},
{"type": "function", "name": "Read"},
"responses",
"none",
),
],
ids=["anthropic", "responses_api"],
)
async def test_pre_call_rewrite_strips_denied_flat_tool_and_forced_choice(
self,
tool_shape: dict[str, object],
tool_choice: dict[str, str],
call_type: CallTypesLiteral,
expected_tool_choice: dict[str, str] | str,
) -> None:
data = {
"model": "claude-sonnet-4-5",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"name": "Bash", **tool_shape}, {"name": "Read", **tool_shape}],
"tool_choice": tool_choice,
}
result = await self._always_on_pre_call("rewrite").async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(),
cache=DualCache(default_in_memory_ttl=1),
data=data,
call_type=call_type,
)
assert [tool["name"] for tool in result["tools"]] == ["Bash"]
assert result["tool_choice"] == expected_tool_choice
def _pre_call(
self,
rules: list[dict[str, str]],
on_disallowed_action: Literal["block", "rewrite"],
default_action: Literal["allow", "deny"] = "allow",
) -> ToolPermissionGuardrail:
return ToolPermissionGuardrail(
guardrail_name=f"request-shapes-{on_disallowed_action}",
rules=rules,
default_action=default_action,
on_disallowed_action=on_disallowed_action,
event_hook=GuardrailEventHooks.pre_call,
default_on=True,
)
async def _run_pre_call(
self, guardrail: ToolPermissionGuardrail, data: dict[str, object], call_type: CallTypesLiteral
) -> dict[str, object]:
return await guardrail.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(),
cache=DualCache(default_in_memory_ttl=1),
data=data,
call_type=call_type,
)
def _namespace(self, *member_names: str) -> dict[str, object]:
return {
"type": "namespace",
"name": "mcp__everything",
"tools": [{"type": "function", "name": name, "parameters": {"type": "object"}} for name in member_names],
}
@pytest.mark.asyncio
async def test_pre_call_blocks_a_denied_tool_inside_a_responses_namespace(self) -> None:
guardrail = self._pre_call(
[{"id": "deny_sum", "tool_name": r"^mcp__everything__get_sum$", "decision": "deny"}], "block"
)
data = {"model": "gpt-5", "input": "add 2 and 3", "tools": [self._namespace("echo", "get_sum")]}
with pytest.raises(HTTPException) as excinfo:
await self._run_pre_call(guardrail, data, "responses")
assert excinfo.value.detail["detection_message"] == "Tool 'mcp__everything__get_sum' denied by rule 'deny_sum'"
@pytest.mark.asyncio
async def test_pre_call_rewrite_drops_denied_namespace_members_and_empty_namespaces(self) -> None:
guardrail = self._pre_call([{"id": "deny_sum", "tool_name": r"^.*get_sum$", "decision": "deny"}], "rewrite")
partly_denied = {**self._namespace("echo", "get_sum"), "name": "mcp__kept"}
data = {"model": "gpt-5", "input": "add", "tools": [partly_denied, self._namespace("get_sum")]}
result = await self._run_pre_call(guardrail, data, "responses")
assert result["tools"] == [{**partly_denied, "tools": partly_denied["tools"][:1]}]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("rule", "tool", "call_type", "detection_message"),
[
(
{"id": "deny_mcp", "tool_type": r"^mcp$", "decision": "deny"},
{"type": "mcp", "server_label": "weather", "server_url": "https://mcp.example.com"},
"responses",
"Tool 'mcp' denied by rule 'deny_mcp'",
),
(
{"id": "allow_bash", "tool_name": r"^Bash$", "decision": "allow"},
{"type": "mcp_toolset", "mcp_server_name": "weather"},
"anthropic_messages",
"Tool 'mcp_toolset' denied by default action",
),
(
{"id": "deny_read", "tool_name": r"^Read$", "decision": "deny"},
{"type": "custom", "custom": {"name": "Read"}},
"acompletion",
"Tool 'Read' denied by rule 'deny_read'",
),
],
ids=["responses_mcp_by_type", "anthropic_mcp_toolset_by_default", "chat_custom_nested_name"],
)
async def test_pre_call_checks_tools_that_carry_no_top_level_name(
self, rule: dict[str, str], tool: dict[str, object], call_type: CallTypesLiteral, detection_message: str
) -> None:
guardrail = self._pre_call([rule], "block", default_action="deny")
data = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "input": "hi", "tools": [tool]}
with pytest.raises(HTTPException) as excinfo:
await self._run_pre_call(guardrail, data, call_type)
assert excinfo.value.detail["detection_message"] == detection_message
@pytest.mark.asyncio
async def test_pre_call_passes_a_tool_no_rule_could_ever_match(self) -> None:
guardrail = self._pre_call(self.rules, "block", default_action="deny")
data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [{"googleSearch": {}}]}
result = await self._run_pre_call(guardrail, data, "acompletion")
assert result["tools"] == [{"googleSearch": {}}]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tools", "tool_choice", "call_type", "expected_tool_choice"),
[
(
[{"type": "custom", "name": "Bash"}, {"type": "custom", "name": "Read"}],
{"type": "custom", "name": "Read"},
"responses",
"none",
),
(
[{"type": "function", "name": "Bash"}, {"type": "function", "name": "Read"}],
{
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": "Read"}, {"type": "function", "name": "Bash"}],
},
"responses",
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "Bash"}]},
),
(
[{"type": "function", "name": "Bash"}, {"type": "function", "name": "Read"}],
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "Read"}]},
"responses",
"none",
),
(
[
{"type": "function", "function": {"name": "Bash"}},
{"type": "function", "function": {"name": "Read"}},
],
{
"type": "allowed_tools",
"allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "Read"}}]},
},
"acompletion",
"none",
),
(
[{"name": "Read", "input_schema": {"type": "object"}}],
{"type": "any"},
"anthropic_messages",
{"type": "none"},
),
(
[
{"name": "Bash", "input_schema": {"type": "object"}},
{"name": "Read", "input_schema": {"type": "object"}},
],
{"type": "any"},
"anthropic_messages",
{"type": "any"},
),
],
ids=[
"responses_custom_choice",
"responses_allowed_tools_keeps_the_rest",
"responses_allowed_tools_all_denied",
"chat_allowed_tools_all_denied",
"anthropic_any_with_no_tools_left",
"anthropic_any_with_a_tool_left",
],
)
async def test_pre_call_rewrite_leaves_no_tool_choice_that_needs_a_removed_tool(
self,
tools: list[dict[str, object]],
tool_choice: dict[str, object],
call_type: CallTypesLiteral,
expected_tool_choice: dict[str, object] | str,
) -> None:
guardrail = self._pre_call(self.rules, "rewrite")
data = {
"model": "m",
"messages": [{"role": "user", "content": "hi"}],
"tools": tools,
"tool_choice": tool_choice,
}
result = await self._run_pre_call(guardrail, data, call_type)
assert result["tool_choice"] == expected_tool_choice
@pytest.mark.asyncio
async def test_pre_call_rewrite_relaxes_a_choice_for_a_tool_a_type_rule_removed(self) -> None:
guardrail = self._pre_call([{"id": "deny_custom", "tool_type": r"^custom$", "decision": "deny"}], "rewrite")
data = {
"model": "gpt-5",
"input": "patch it",
"tools": [{"type": "function", "name": "Bash"}, {"type": "custom", "name": "apply_patch"}],
"tool_choice": {"type": "custom", "name": "apply_patch"},
}
result = await self._run_pre_call(guardrail, data, "responses")
assert result["tools"] == [{"type": "function", "name": "Bash"}]
assert result["tool_choice"] == "none"
@pytest.mark.asyncio
async def test_denied_anthropic_tool_use_is_blocked(self):
response = self._response({"type": "text", "text": "reading"}, self._tool_use("Read"))

View file

@ -2179,6 +2179,31 @@ class TestResponsesWSFirstFrameModelAuth:
"response": {"input": "hi [guarded]", "tools": tools[:1]},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", [True, False])
async def test_frame_guard_forwards_the_tool_choice_a_guardrail_rewrote(self, nested: bool):
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only):
return {**data, "tools": [], "tool_choice": "none"}
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
body = {
"input": "hi",
"tools": [{"type": "function", "name": "a"}],
"tool_choice": {"type": "function", "name": "a"},
}
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
guarded = json.loads((await guard(json.dumps(frame))).frame)
sent = guarded["response"] if nested else guarded
assert (sent["tools"], sent["tool_choice"]) == ([], "none")
@pytest.mark.asyncio
async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self):
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard