mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): prune tool_choice to surviving tools
When a guardrail or tool filter drops a tool, trim tool_choice down to the tools that survived, falling back to auto (or none when nothing survives). Covers named, custom, allowed_tools and Anthropic shapes. Adds 21 regression tests.
This commit is contained in:
parent
032342d2b7
commit
65bc23325a
2 changed files with 251 additions and 0 deletions
|
|
@ -1706,6 +1706,58 @@ def _proxy_hook_factory(hook_name: str) -> Callable[..., CustomLogger]:
|
|||
return get_proxy_hook(hook_name)
|
||||
|
||||
|
||||
def _tool_definition_name(tool: object) -> str | None:
|
||||
if not isinstance(tool, Mapping):
|
||||
return None
|
||||
tool_type: Final = tool.get("type")
|
||||
if isinstance(tool_type, str):
|
||||
nested: Final = tool.get(tool_type)
|
||||
if isinstance(nested, Mapping):
|
||||
nested_name: Final = nested.get("name")
|
||||
if isinstance(nested_name, str) and nested_name:
|
||||
return nested_name
|
||||
name: Final = tool.get("name")
|
||||
return name if isinstance(name, str) and name else None
|
||||
|
||||
|
||||
def prune_tool_choice_to_surviving_tools(tools: object, tool_choice: object) -> object:
|
||||
if not isinstance(tool_choice, Mapping):
|
||||
return tool_choice
|
||||
if tools is None or isinstance(tools, (str, bytes)) or not isinstance(tools, Sequence):
|
||||
return tool_choice
|
||||
surviving: Final = frozenset(tool_name for tool in tools if (tool_name := _tool_definition_name(tool)) is not None)
|
||||
choice_type: Final = tool_choice.get("type")
|
||||
if choice_type in ("function", "custom", "tool"):
|
||||
choice_name: Final = _tool_definition_name(tool_choice)
|
||||
if choice_name is None or choice_name in surviving:
|
||||
return tool_choice
|
||||
if choice_type == "tool":
|
||||
return {"type": "auto"} if surviving else {"type": "none"}
|
||||
return "auto" if surviving else "none"
|
||||
if choice_type == "allowed_tools":
|
||||
inner: Final = tool_choice.get("allowed_tools")
|
||||
entries: Final = inner.get("tools") if isinstance(inner, Mapping) else tool_choice.get("tools")
|
||||
if entries is None or isinstance(entries, (str, bytes)) or not isinstance(entries, Sequence):
|
||||
return tool_choice
|
||||
kept: Final = [
|
||||
entry
|
||||
for entry in entries
|
||||
if (entry if isinstance(entry, str) else _tool_definition_name(entry)) in surviving
|
||||
]
|
||||
if kept:
|
||||
if isinstance(inner, Mapping):
|
||||
nested_inner: Final = dict(inner)
|
||||
nested_inner["tools"] = kept
|
||||
nested_choice: Final = dict(tool_choice)
|
||||
nested_choice["allowed_tools"] = nested_inner
|
||||
return nested_choice
|
||||
flat_choice: Final = dict(tool_choice)
|
||||
flat_choice["tools"] = kept
|
||||
return flat_choice
|
||||
return "auto" if surviving else "none"
|
||||
return tool_choice
|
||||
|
||||
|
||||
class ProxyLogging:
|
||||
"""
|
||||
Logging/Custom Handlers for proxy.
|
||||
|
|
@ -3121,6 +3173,8 @@ class ProxyLogging:
|
|||
|
||||
if data is not None:
|
||||
self._process_guardrail_metadata(data)
|
||||
if "tool_choice" in data:
|
||||
data["tool_choice"] = prune_tool_choice_to_surviving_tools(data.get("tools"), data["tool_choice"])
|
||||
|
||||
return data
|
||||
except SensitiveDataRouteException as e:
|
||||
|
|
|
|||
|
|
@ -3714,3 +3714,200 @@ async def test_evict_config_param_clears_the_local_layer_and_survives_a_redis_ou
|
|||
await evict_config_param("router_settings", cache=cache)
|
||||
|
||||
assert await cache.in_memory_cache.async_get_cache(_config_cache_key("router_settings")) is None
|
||||
|
||||
|
||||
def _function_tool(name):
|
||||
return {"type": "function", "function": {"name": name}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tools,tool_choice,expected",
|
||||
[
|
||||
(
|
||||
[_function_tool("get_time"), _function_tool("get_weather")],
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
),
|
||||
(
|
||||
[_function_tool("get_time")],
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
"auto",
|
||||
),
|
||||
(
|
||||
[],
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
"none",
|
||||
),
|
||||
(
|
||||
None,
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
),
|
||||
(
|
||||
[_function_tool("get_time")],
|
||||
{"type": "custom", "custom": {"name": "get_weather"}},
|
||||
"auto",
|
||||
),
|
||||
(
|
||||
[_function_tool("get_time"), _function_tool("get_weather")],
|
||||
{"type": "custom", "custom": {"name": "get_weather"}},
|
||||
{"type": "custom", "custom": {"name": "get_weather"}},
|
||||
),
|
||||
(
|
||||
[_function_tool("get_time")],
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "auto",
|
||||
"tools": [_function_tool("get_weather"), _function_tool("get_time")],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "auto", "tools": [_function_tool("get_time")]},
|
||||
},
|
||||
),
|
||||
(
|
||||
[_function_tool("get_time")],
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "auto", "tools": [_function_tool("get_weather")]},
|
||||
},
|
||||
"auto",
|
||||
),
|
||||
(
|
||||
[],
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "required", "tools": [_function_tool("get_weather")]},
|
||||
},
|
||||
"none",
|
||||
),
|
||||
(
|
||||
[_function_tool("get_time")],
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [_function_tool("get_weather"), _function_tool("get_time")],
|
||||
},
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [_function_tool("get_time")],
|
||||
},
|
||||
),
|
||||
(
|
||||
[{"name": "get_time", "input_schema": {}}],
|
||||
{"type": "tool", "name": "get_weather"},
|
||||
{"type": "auto"},
|
||||
),
|
||||
(
|
||||
[{"name": "get_time", "input_schema": {}}, {"name": "get_weather", "input_schema": {}}],
|
||||
{"type": "tool", "name": "get_weather"},
|
||||
{"type": "tool", "name": "get_weather"},
|
||||
),
|
||||
([_function_tool("get_time")], "auto", "auto"),
|
||||
([_function_tool("get_time")], "none", "none"),
|
||||
([_function_tool("get_time")], {"type": "auto"}, {"type": "auto"}),
|
||||
([_function_tool("get_time")], {"type": "other"}, {"type": "other"}),
|
||||
([_function_tool("get_time")], None, None),
|
||||
(
|
||||
[_function_tool("get_time")],
|
||||
{"type": "function", "function": {}},
|
||||
{"type": "function", "function": {}},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_prune_tool_choice_to_surviving_tools(tools, tool_choice, expected):
|
||||
from litellm.proxy.utils import prune_tool_choice_to_surviving_tools
|
||||
|
||||
assert prune_tool_choice_to_surviving_tools(tools, tool_choice) == expected
|
||||
|
||||
|
||||
def test_prune_tool_choice_to_surviving_tools_keeps_surviving_choice_unchanged():
|
||||
from litellm.proxy.utils import prune_tool_choice_to_surviving_tools
|
||||
|
||||
choice = {"type": "function", "function": {"name": "get_weather"}}
|
||||
tools = [_function_tool("get_time"), _function_tool("get_weather")]
|
||||
|
||||
assert prune_tool_choice_to_surviving_tools(tools, choice) is choice
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_prunes_tool_choice_after_guardrail_removes_tool():
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
class RemovingGuardrail(CustomGuardrail):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
guardrail_name="tool-pruner",
|
||||
event_hook=GuardrailEventHooks.pre_call,
|
||||
default_on=True,
|
||||
)
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
data["tools"] = [tool for tool in data.get("tools") or [] if tool["function"]["name"] != "get_weather"]
|
||||
return data
|
||||
|
||||
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
|
||||
original_callbacks = litellm.callbacks.copy() if litellm.callbacks else []
|
||||
|
||||
try:
|
||||
litellm.callbacks = [RemovingGuardrail()]
|
||||
|
||||
result = await proxy_logging.pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="<redacted>", user_id="test_user"),
|
||||
data={
|
||||
"model": "gpt-5.5",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [_function_tool("get_time"), _function_tool("get_weather")],
|
||||
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
|
||||
},
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert result["tools"] == [_function_tool("get_time")]
|
||||
assert result["tool_choice"] == "auto"
|
||||
finally:
|
||||
litellm.callbacks = original_callbacks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_keeps_tool_choice_naming_surviving_tool():
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
class RemovingGuardrail(CustomGuardrail):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
guardrail_name="tool-pruner",
|
||||
event_hook=GuardrailEventHooks.pre_call,
|
||||
default_on=True,
|
||||
)
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
data["tools"] = [tool for tool in data.get("tools") or [] if tool["function"]["name"] != "get_weather"]
|
||||
return data
|
||||
|
||||
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
|
||||
original_callbacks = litellm.callbacks.copy() if litellm.callbacks else []
|
||||
|
||||
try:
|
||||
litellm.callbacks = [RemovingGuardrail()]
|
||||
|
||||
result = await proxy_logging.pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="<redacted>", user_id="test_user"),
|
||||
data={
|
||||
"model": "gpt-5.5",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [_function_tool("get_time"), _function_tool("get_weather")],
|
||||
"tool_choice": {"type": "function", "function": {"name": "get_time"}},
|
||||
},
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert result["tools"] == [_function_tool("get_time")]
|
||||
assert result["tool_choice"] == {"type": "function", "function": {"name": "get_time"}}
|
||||
finally:
|
||||
litellm.callbacks = original_callbacks
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue