mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 2de6ddf153 into f4a2e2b3af
This commit is contained in:
commit
ba684d1856
12 changed files with 2775 additions and 119 deletions
|
|
@ -8,6 +8,7 @@ import time
|
|||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from functools import lru_cache, partial
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
|
||||
|
|
@ -544,6 +545,59 @@ def mapped_gemini_speech_config(
|
|||
}
|
||||
|
||||
|
||||
def _as_chat_tool(member: JsonValue) -> JsonValue:
|
||||
if not isinstance(member, dict) or "function" in member or "name" not in member:
|
||||
return member
|
||||
return {
|
||||
"type": member.get("type", "function"),
|
||||
"function": {key: value for key, value in member.items() if key != "type"},
|
||||
}
|
||||
|
||||
|
||||
def _namespace_members_or_tool(tool: JsonValue) -> tuple[JsonValue, ...]:
|
||||
if not isinstance(tool, dict) or tool.get("type") != "namespace":
|
||||
return (tool,)
|
||||
members: Final = tool.get("tools")
|
||||
return tuple(map(_as_chat_tool, members)) if isinstance(members, list) else ()
|
||||
|
||||
|
||||
def _declared_function_name(tool: JsonValue) -> str | None:
|
||||
if not isinstance(tool, dict):
|
||||
return None
|
||||
function: Final = tool.get("function")
|
||||
name: Final = function.get("name") if isinstance(function, dict) else tool.get("name")
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
|
||||
def _is_namespace(tool: JsonValue) -> bool:
|
||||
return isinstance(tool, dict) and tool.get("type") == "namespace"
|
||||
|
||||
|
||||
def _first_declarations(tools: tuple[JsonValue, ...]) -> tuple[JsonValue, ...]:
|
||||
names: Final = tuple(map(_declared_function_name, tools))
|
||||
first_index: Final = {name: index for index, name in reversed(tuple(enumerate(names)))}
|
||||
return tuple(tool for index, tool in enumerate(tools) if names[index] is None or first_index[names[index]] == index)
|
||||
|
||||
|
||||
def _with_namespace_members_inlined(tools: JsonValue) -> JsonValue:
|
||||
if not isinstance(tools, list) or not any(map(_is_namespace, tools)):
|
||||
return tools
|
||||
return list(_first_declarations(tuple(chain.from_iterable(map(_namespace_members_or_tool, tools)))))
|
||||
|
||||
|
||||
def _forced_function_names(name: str, tools: JsonValue) -> tuple[str, ...]:
|
||||
candidates: Final = tools if isinstance(tools, list) else []
|
||||
if any(not _is_namespace(tool) and _declared_function_name(tool) == name for tool in candidates):
|
||||
return (name,)
|
||||
named: Final = next((tool for tool in candidates if _is_namespace(tool) and _member_name_of(tool) == name), None)
|
||||
members: Final = map(_declared_function_name, _namespace_members_or_tool(named)) if named is not None else ()
|
||||
return tuple(dict.fromkeys(member for member in members if member is not None)) or (name,)
|
||||
|
||||
|
||||
def _member_name_of(tool: JsonValue) -> JsonValue:
|
||||
return tool.get("name") if isinstance(tool, dict) else None
|
||||
|
||||
|
||||
class VertexAIBaseConfig:
|
||||
def get_mapped_special_auth_params(self) -> dict:
|
||||
"""
|
||||
|
|
@ -815,7 +869,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
return supported_params
|
||||
|
||||
def map_tool_choice_values(self, model: str, tool_choice: str | dict) -> ToolConfig | None:
|
||||
def map_tool_choice_values(self, model: str, tool_choice: str | dict, tools: JsonValue = None) -> ToolConfig | None:
|
||||
if tool_choice == "none":
|
||||
return ToolConfig(functionCallingConfig=FunctionCallingConfig(mode="NONE"))
|
||||
elif tool_choice == "required":
|
||||
|
|
@ -825,7 +879,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif isinstance(tool_choice, dict):
|
||||
# only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
|
||||
name: Final = tool_choice.get("function", {}).get("name", "")
|
||||
return ToolConfig(functionCallingConfig=FunctionCallingConfig(mode="ANY", allowed_function_names=[name]))
|
||||
return ToolConfig(
|
||||
functionCallingConfig=FunctionCallingConfig(
|
||||
mode="ANY", allowed_function_names=list(_forced_function_names(name, tools))
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=f"VertexAI doesn't support tool_choice={tool_choice}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.",
|
||||
|
|
@ -1099,7 +1157,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# remove 'strict' from tools
|
||||
remove_strict_from_schema(schema_value)
|
||||
|
||||
for tool in cast(list[dict], schema_value): # cast-ok: a deep copy keeps the caller's tool shape
|
||||
inlined_tools: Final = _with_namespace_members_inlined(schema_value)
|
||||
for tool in cast(list[dict], inlined_tools): # cast-ok: a deep copy keeps the caller's tool shape
|
||||
openai_function_object: ChatCompletionToolParamFunctionChunk | None = None
|
||||
if "function" in tool: # tools list
|
||||
_openai_function_object = ChatCompletionToolParamFunctionChunk(**tool["function"])
|
||||
|
|
@ -1742,6 +1801,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
_tool_choice_value = self.map_tool_choice_values(
|
||||
model=model,
|
||||
tool_choice=value,
|
||||
tools=non_default_params.get("tools", non_default_params.get("functions")),
|
||||
)
|
||||
if _tool_choice_value is not None:
|
||||
optional_params["tool_choice"] = _tool_choice_value
|
||||
|
|
|
|||
|
|
@ -19977,7 +19977,7 @@
|
|||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Regex pattern applied to the tool's function name",
|
||||
"description": "Regex pattern applied to the tool's name, or to its type when the tool has no name",
|
||||
"title": "Tool Name"
|
||||
},
|
||||
"tool_type": {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -2162,7 +2162,7 @@ _WS_FRAME: Final = TypeAdapter(dict[str, object])
|
|||
_GUARDRAIL_INFORMATION: Final = TypeAdapter(list[dict[str, object]])
|
||||
_PII_TOKENS: Final = TypeAdapter(dict[str, str])
|
||||
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
|
||||
_GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools")
|
||||
_GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools", "tool_choice", "extra_body")
|
||||
_CLIENT_METADATA_KEYS: Final = ("metadata", "litellm_metadata")
|
||||
_CLIENT_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config", "policies")
|
||||
_METADATA_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config")
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class ToolPermissionRule(LiteLLMBaseModel):
|
|||
id: str = Field(description="Unique identifier for the rule")
|
||||
tool_name: str | None = Field(
|
||||
default=None,
|
||||
description="Regex pattern applied to the tool's function name",
|
||||
description="Regex pattern applied to the tool's name, or to its type when the tool has no name",
|
||||
)
|
||||
tool_type: str | None = Field(
|
||||
default=None,
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@
|
|||
- {id: guardrail.ibm_guardrails.post_call.blocks, module: guardrail, tier: P1, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/ibm_guardrails/ibm_detector.py", rationale: "Output policy validation"}
|
||||
- {id: guardrail.semantic_guard.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/semantic_guard", rationale: "Semantic policy compliance"}
|
||||
- {id: guardrail.block_code_execution.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/block_code_execution", rationale: "Code-injection prevention"}
|
||||
- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"}
|
||||
- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"}
|
||||
- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"}
|
||||
- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"}
|
||||
- {id: guardrail.microsoft_purview.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/microsoft_purview/purview_dlp.py", rationale: "DLP sensitive-data disclosure"}
|
||||
- {id: guardrail.headroom.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/headroom/headroom.py", rationale: "Anomaly detection threshold"}
|
||||
- {id: guardrail.generic_guardrail_api.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py", rationale: "Vendor-agnostic custom API"}
|
||||
|
|
|
|||
|
|
@ -19,10 +19,9 @@ verdict comes from the proxy itself.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse, UnknownApiError
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Subject, meta
|
||||
|
|
@ -31,14 +30,29 @@ from guardrails_client import (
|
|||
ToolPermissionParamsBody,
|
||||
ToolPermissionRuleBody,
|
||||
poll_until_blocked,
|
||||
poll_until_blocked_stream,
|
||||
poll_until_guardrail_applied,
|
||||
)
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatResponse, ChatTool, ChatToolFunction
|
||||
from models import (
|
||||
AnthropicCustomTool,
|
||||
AnthropicMessagesBody,
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicToolChoice,
|
||||
ChatMessage,
|
||||
ChatResponse,
|
||||
ChatTool,
|
||||
ChatToolFunction,
|
||||
JsonSchemaProperty,
|
||||
ResponsesFunctionTool,
|
||||
ResponsesToolBody,
|
||||
ResponsesToolOutput,
|
||||
ToolInputSchema,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
pytestmark: Final = pytest.mark.e2e
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
MODEL: Final = "gemini-2.5-flash"
|
||||
|
||||
#: The one tool the guardrail permits, and one it does not. Both are declared by
|
||||
#: the caller in the request body; the guardrail reads them there.
|
||||
|
|
@ -63,11 +77,15 @@ DENIED_TOOL: Final = ChatTool(
|
|||
|
||||
TOOL_PROMPT: Final = "What is the weather in Paris right now?"
|
||||
|
||||
type FlatToolEndpoint = Literal["messages", "responses"]
|
||||
ALLOWED_TOOL_SCHEMA: Final = ToolInputSchema(properties={"city": JsonSchemaProperty(type="string")}, required=["city"])
|
||||
DENIED_TOOL_SCHEMA: Final = ToolInputSchema()
|
||||
|
||||
|
||||
def _register_tool_permission(client: GuardrailsClient, resources: ResourceManager, *, name: str) -> None:
|
||||
"""Allow-list exactly one tool: everything else falls to `default_action=deny`
|
||||
and, with `on_disallowed_action=block`, is rejected outright."""
|
||||
guardrail_id = client.register(
|
||||
guardrail_id: Final = client.register(
|
||||
name,
|
||||
ToolPermissionParamsBody(
|
||||
mode="pre_call",
|
||||
|
|
@ -100,6 +118,57 @@ def _tool_call_names(response: ChatResponse) -> tuple[str, ...]:
|
|||
)
|
||||
|
||||
|
||||
def _send_flat_tool(
|
||||
client: GuardrailsClient,
|
||||
key: str,
|
||||
endpoint: FlatToolEndpoint,
|
||||
*,
|
||||
guardrail: str,
|
||||
tool: ChatTool,
|
||||
schema: ToolInputSchema,
|
||||
require_tool: bool = False,
|
||||
) -> StreamingResponse:
|
||||
name: Final = tool.function.name
|
||||
match endpoint:
|
||||
case "messages":
|
||||
return client.proxy.transport.send(
|
||||
"/v1/messages",
|
||||
headers=client.proxy.transport.bearer(key),
|
||||
json=AnthropicMessagesBody(
|
||||
model=MODEL,
|
||||
messages=[ChatMessage(role="user", content=TOOL_PROMPT)],
|
||||
max_tokens=128,
|
||||
tools=[
|
||||
AnthropicCustomTool(name=name, description=tool.function.description or "", input_schema=schema)
|
||||
],
|
||||
tool_choice=AnthropicToolChoice(type="any") if require_tool else None,
|
||||
guardrails=[guardrail],
|
||||
),
|
||||
)
|
||||
case "responses":
|
||||
return client.proxy.transport.send(
|
||||
"/v1/responses",
|
||||
headers=client.proxy.transport.bearer(key),
|
||||
json=ResponsesToolBody(
|
||||
model=MODEL,
|
||||
input=TOOL_PROMPT,
|
||||
tools=[ResponsesFunctionTool(name=name, description=tool.function.description, parameters=schema)],
|
||||
tool_choice="required" if require_tool else None,
|
||||
guardrails=[guardrail],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _flat_tool_call_names(endpoint: FlatToolEndpoint, body: str) -> tuple[str, ...]:
|
||||
match endpoint:
|
||||
case "messages":
|
||||
content: Final = AnthropicMessagesResponse.model_validate_json(body).content or []
|
||||
return tuple(block.name for block in content if block.type == "tool_use" and block.name)
|
||||
case "responses":
|
||||
output: Final = ResponsesToolOutput.model_validate_json(body).output
|
||||
return tuple(item.name for item in output if item.type == "function_call" and item.name)
|
||||
|
||||
|
||||
class TestToolPermissionPreCall:
|
||||
@pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -118,10 +187,10 @@ class TestToolPermissionPreCall:
|
|||
rejected with a 400 that names the denied tool. An unauthorized tool that
|
||||
merely reaches the model is the whole failure mode this guardrail exists
|
||||
to prevent, so a 200 here is a hard failure."""
|
||||
name = f"e2e-toolperm-block-{unique_marker()}"
|
||||
name: Final = f"e2e-toolperm-block-{unique_marker()}"
|
||||
_register_tool_permission(client, resources, name=name)
|
||||
|
||||
result = poll_until_blocked(
|
||||
result: Final = poll_until_blocked(
|
||||
lambda: client.chat(
|
||||
scoped_key,
|
||||
MODEL,
|
||||
|
|
@ -161,10 +230,10 @@ class TestToolPermissionPreCall:
|
|||
and the model calls it. Without the header check a guardrail that never
|
||||
attached would pass this test for the wrong reason, so the 200 alone is
|
||||
not the contract."""
|
||||
name = f"e2e-toolperm-allow-{unique_marker()}"
|
||||
name: Final = f"e2e-toolperm-allow-{unique_marker()}"
|
||||
_register_tool_permission(client, resources, name=name)
|
||||
|
||||
outcome = poll_until_guardrail_applied(
|
||||
outcome: Final = poll_until_guardrail_applied(
|
||||
lambda: client.chat_raw(
|
||||
scoped_key,
|
||||
MODEL,
|
||||
|
|
@ -178,13 +247,117 @@ class TestToolPermissionPreCall:
|
|||
)
|
||||
|
||||
assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}"
|
||||
applied = _applied_guardrails(outcome)
|
||||
applied: Final = _applied_guardrails(outcome)
|
||||
assert name in applied, (
|
||||
"the allowed call must carry x-litellm-applied-guardrails naming the guardrail; "
|
||||
f"without it the 200 only proves the guardrail never ran. Got {applied!r}"
|
||||
)
|
||||
|
||||
called = _tool_call_names(ChatResponse.model_validate_json(outcome.body))
|
||||
called: Final = _tool_call_names(ChatResponse.model_validate_json(outcome.body))
|
||||
assert called == (ALLOWED_TOOL.function.name,), (
|
||||
f"the served call must carry one tool call for the permitted tool, got {called!r}: {outcome.body[:400]}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint",
|
||||
[
|
||||
pytest.param(
|
||||
"messages",
|
||||
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["messages"]),
|
||||
),
|
||||
pytest.param(
|
||||
"responses",
|
||||
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["responses"]),
|
||||
),
|
||||
],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
capabilities=(Capability.FUNCTION_CALLING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_pre_call_blocks_flat_tool_outside_the_allow_list(
|
||||
self,
|
||||
client: GuardrailsClient,
|
||||
resources: ResourceManager,
|
||||
scoped_key: str,
|
||||
endpoint: FlatToolEndpoint,
|
||||
) -> None:
|
||||
name: Final = f"e2e-toolperm-block-{endpoint}-{unique_marker()}"
|
||||
_register_tool_permission(client, resources, name=name)
|
||||
|
||||
outcome: Final = poll_until_blocked_stream(
|
||||
lambda: _send_flat_tool(
|
||||
client, scoped_key, endpoint, guardrail=name, tool=DENIED_TOOL, schema=DENIED_TOOL_SCHEMA
|
||||
)
|
||||
)
|
||||
|
||||
assert outcome.status_code == 400, (
|
||||
f"tool_permission let a tool outside the allow-list through /{endpoint}; "
|
||||
f"got {outcome.status_code}: {outcome.body[:400]}"
|
||||
)
|
||||
assert DENIED_TOOL.function.name in outcome.body, (
|
||||
f"the block must name the denied tool so the caller can fix the request; got: {outcome.body[:400]}"
|
||||
)
|
||||
assert "Violated guardrail policy" in outcome.body, (
|
||||
f"the block must be a guardrail verdict, got: {outcome.body[:400]}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint",
|
||||
[
|
||||
pytest.param(
|
||||
"messages",
|
||||
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["messages"]),
|
||||
),
|
||||
pytest.param(
|
||||
"responses",
|
||||
marks=pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["responses"]),
|
||||
),
|
||||
],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
capabilities=(Capability.FUNCTION_CALLING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_pre_call_allows_permitted_flat_tool(
|
||||
self,
|
||||
client: GuardrailsClient,
|
||||
resources: ResourceManager,
|
||||
scoped_key: str,
|
||||
endpoint: FlatToolEndpoint,
|
||||
) -> None:
|
||||
name: Final = f"e2e-toolperm-allow-{endpoint}-{unique_marker()}"
|
||||
_register_tool_permission(client, resources, name=name)
|
||||
|
||||
outcome: Final = poll_until_guardrail_applied(
|
||||
lambda: _send_flat_tool(
|
||||
client,
|
||||
scoped_key,
|
||||
endpoint,
|
||||
guardrail=name,
|
||||
tool=ALLOWED_TOOL,
|
||||
schema=ALLOWED_TOOL_SCHEMA,
|
||||
require_tool=True,
|
||||
),
|
||||
name,
|
||||
)
|
||||
|
||||
assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}"
|
||||
applied: Final = _applied_guardrails(outcome)
|
||||
assert name in applied, (
|
||||
f"the allowed call must carry x-litellm-applied-guardrails naming the guardrail; got {applied!r}"
|
||||
)
|
||||
called: Final = _flat_tool_call_names(endpoint, outcome.body)
|
||||
assert called == (ALLOWED_TOOL.function.name,), (
|
||||
f"the served call must carry one tool call for the permitted tool, got {called!r}: {outcome.body[:400]}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -653,6 +653,35 @@ class ResponsesStreamBody(BaseModel):
|
|||
cache: dict[str, bool] | None = {"no-cache": True}
|
||||
|
||||
|
||||
class ResponsesFunctionTool(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
type: Literal["function"] = "function"
|
||||
name: str
|
||||
description: str | None = None
|
||||
parameters: ToolInputSchema
|
||||
|
||||
|
||||
class ResponsesToolBody(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
model: str
|
||||
input: str
|
||||
tools: list[ResponsesFunctionTool]
|
||||
tool_choice: Literal["required"] | None = None
|
||||
guardrails: list[str]
|
||||
max_output_tokens: int = 128
|
||||
|
||||
|
||||
class ResponsesOutputItem(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
type: str
|
||||
name: str | None = None
|
||||
|
||||
|
||||
class ResponsesToolOutput(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
output: list[ResponsesOutputItem]
|
||||
|
||||
|
||||
class CountTokensBody(BaseModel):
|
||||
"""POST /v1/messages/count_tokens body: the /v1/messages shape minus
|
||||
max_tokens (the endpoint only counts the prompt)."""
|
||||
|
|
|
|||
|
|
@ -1785,6 +1785,110 @@ def test_vertex_ai_map_tools():
|
|||
assert tools == new_tools
|
||||
|
||||
|
||||
def test_vertex_ai_map_tools_declares_a_namespace_by_its_members() -> None:
|
||||
namespace = {
|
||||
"type": "namespace",
|
||||
"name": "github",
|
||||
"tools": [
|
||||
{"type": "function", "name": "create_issue", "parameters": {"type": "object", "properties": {}}},
|
||||
{"type": "function", "function": {"name": "close_issue"}},
|
||||
],
|
||||
}
|
||||
|
||||
tools = VertexGeminiConfig().map_function(value=[namespace], optional_params={})
|
||||
|
||||
assert [declaration["name"] for declaration in tools[0]["function_declarations"]] == [
|
||||
"create_issue",
|
||||
"close_issue",
|
||||
]
|
||||
|
||||
|
||||
def test_vertex_ai_map_tools_declares_a_namespace_member_like_the_same_chat_tool() -> None:
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"repo": {"$ref": "#/$defs/Repo"}, "label": {"type": ["string", "null"]}},
|
||||
"$defs": {"Repo": {"type": "string"}},
|
||||
}
|
||||
function = {"name": "create_issue", "description": "Open an issue", "parameters": parameters}
|
||||
namespace = {"type": "namespace", "name": "github", "tools": [{"type": "function", **function}]}
|
||||
|
||||
from_namespace = VertexGeminiConfig().map_function(value=[namespace], optional_params={})
|
||||
from_chat_tool = VertexGeminiConfig().map_function(
|
||||
value=[{"type": "function", "function": function}], optional_params={}
|
||||
)
|
||||
|
||||
assert from_namespace == from_chat_tool
|
||||
|
||||
|
||||
_GET_WEATHER = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tools",
|
||||
[
|
||||
[
|
||||
{"type": "namespace", "name": "forecast", "tools": [_GET_WEATHER]},
|
||||
{"type": "namespace", "name": "weather", "tools": [_GET_WEATHER]},
|
||||
],
|
||||
[
|
||||
{"type": "function", "function": {k: v for k, v in _GET_WEATHER.items() if k != "type"}},
|
||||
{"type": "namespace", "name": "weather", "tools": [_GET_WEATHER]},
|
||||
],
|
||||
],
|
||||
ids=["namespaces_share_a_member", "member_repeats_a_top_level_tool"],
|
||||
)
|
||||
def test_vertex_ai_map_tools_declares_a_repeated_member_name_once_and_never_a_namespace_name(
|
||||
tools: list[dict],
|
||||
) -> None:
|
||||
mapped = VertexGeminiConfig().map_function(value=tools, optional_params={})
|
||||
|
||||
assert [declaration["name"] for declaration in mapped[0]["function_declarations"]] == ["get_weather"]
|
||||
|
||||
|
||||
def test_vertex_ai_map_tools_keeps_repeated_chat_tools_when_no_namespace_is_sent() -> None:
|
||||
function = {k: v for k, v in _GET_WEATHER.items() if k != "type"}
|
||||
|
||||
mapped = VertexGeminiConfig().map_function(
|
||||
value=[{"type": "function", "function": function}, {"type": "function", "function": function}],
|
||||
optional_params={},
|
||||
)
|
||||
|
||||
assert [declaration["name"] for declaration in mapped[0]["function_declarations"]] == ["get_weather", "get_weather"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("forced_name", "other_tools", "allowed_function_names"),
|
||||
[
|
||||
("github", [], ["create_issue", "close_issue"]),
|
||||
("close_issue", [], ["close_issue"]),
|
||||
("github", [{"type": "function", "function": {"name": "github"}}], ["github"]),
|
||||
],
|
||||
ids=["names_the_namespace", "names_a_member", "names_a_function_sharing_the_namespace_name"],
|
||||
)
|
||||
def test_vertex_ai_tool_choice_forces_only_functions_gemini_is_sent(
|
||||
forced_name: str, other_tools: list[dict[str, object]], allowed_function_names: list[str]
|
||||
) -> None:
|
||||
namespace = {
|
||||
"type": "namespace",
|
||||
"name": "github",
|
||||
"tools": [{"type": "function", "name": "create_issue"}, {"type": "function", "name": "close_issue"}],
|
||||
}
|
||||
|
||||
mapped = VertexGeminiConfig().map_openai_params(
|
||||
non_default_params={
|
||||
"tools": [namespace, *other_tools],
|
||||
"tool_choice": {"type": "function", "function": {"name": forced_name}},
|
||||
},
|
||||
optional_params={},
|
||||
model="gemini-3.8-flash",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
declared = [declaration["name"] for declaration in mapped["tools"][0]["function_declarations"]]
|
||||
assert mapped["tool_choice"]["functionCallingConfig"]["allowed_function_names"] == allowed_function_names
|
||||
assert set(allowed_function_names) <= set(declared)
|
||||
|
||||
|
||||
def test_vertex_ai_map_tool_with_anyof():
|
||||
"""
|
||||
Related issue: https://github.com/BerriAI/litellm/issues/11164
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -2556,6 +2556,66 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
"response": {"input": "hi [guarded]", "tools": tools[:1]},
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [True, False])
|
||||
async def test_frame_guard_forwards_the_tool_choice_a_guardrail_rewrote(self, nested: bool):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
||||
async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only):
|
||||
return {**data, "tools": [], "tool_choice": "none"}
|
||||
|
||||
proxy_logging_obj = MagicMock(during_call_hook=AsyncMock())
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
body = {
|
||||
"input": "hi",
|
||||
"tools": [{"type": "function", "name": "a"}],
|
||||
"tool_choice": {"type": "function", "name": "a"},
|
||||
}
|
||||
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
|
||||
|
||||
guarded = json.loads((await guard(json.dumps(frame))).frame)
|
||||
|
||||
sent = guarded["response"] if nested else guarded
|
||||
assert (sent["tools"], sent["tool_choice"]) == ([], "none")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [True, False])
|
||||
async def test_frame_guard_forwards_the_extra_body_tools_a_rewrite_guardrail_stripped(self, nested: bool):
|
||||
from litellm.proxy.guardrails.guardrail_hooks.tool_permission import ToolPermissionGuardrail
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
guardrail = ToolPermissionGuardrail(
|
||||
guardrail_name="ws-rewrite",
|
||||
rules=[{"id": "deny_shell", "tool_name": r"^shell$", "decision": "deny"}],
|
||||
default_action="allow",
|
||||
on_disallowed_action="rewrite",
|
||||
event_hook=GuardrailEventHooks.pre_call,
|
||||
default_on=True,
|
||||
)
|
||||
|
||||
async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only):
|
||||
return await guardrail.async_pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict, cache=MagicMock(), data=data, call_type=call_type
|
||||
)
|
||||
|
||||
proxy_logging_obj = MagicMock(during_call_hook=AsyncMock())
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
weather = {"type": "function", "name": "get_weather"}
|
||||
body = {"input": "hi", "extra_body": {"tools": [weather, {"type": "function", "name": "shell"}]}}
|
||||
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
|
||||
|
||||
guarded = json.loads((await guard(json.dumps(frame))).frame)
|
||||
|
||||
sent = guarded["response"] if nested else guarded
|
||||
assert sent["extra_body"] == {"tools": [weather]}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
|
@ -2991,7 +3051,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
assert dict(guarded.pii_tokens) == {"<EMAIL_ADDRESS_1>": "alice@example.com"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("field", ["input", "instructions", "tools"])
|
||||
@pytest.mark.parametrize("field", ["input", "instructions", "tools", "tool_choice", "extra_body"])
|
||||
async def test_frame_guard_rejects_and_bills_a_guarded_field_beside_a_nested_response(self, field: str):
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -47962,7 +47962,7 @@ export interface components {
|
|||
id: string;
|
||||
/**
|
||||
* Tool Name
|
||||
* @description Regex pattern applied to the tool's function name
|
||||
* @description Regex pattern applied to the tool's name, or to its type when the tool has no name
|
||||
*/
|
||||
tool_name?: string | null;
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue