fix(proxy): prune tool_choice to surviving tools

When a guardrail or tool filter drops a tool, trim tool_choice down to
the tools that survived, falling back to auto (or none when nothing
survives). Covers named, custom, allowed_tools and Anthropic shapes.
Adds 21 regression tests.
This commit is contained in:
Muhammad Mirza 2026-10-29 16:46:36 +07:00
parent 032342d2b7
commit 65bc23325a
2 changed files with 251 additions and 0 deletions

View file

@ -1706,6 +1706,58 @@ def _proxy_hook_factory(hook_name: str) -> Callable[..., CustomLogger]:
return get_proxy_hook(hook_name)
def _tool_definition_name(tool: object) -> str | None:
if not isinstance(tool, Mapping):
return None
tool_type: Final = tool.get("type")
if isinstance(tool_type, str):
nested: Final = tool.get(tool_type)
if isinstance(nested, Mapping):
nested_name: Final = nested.get("name")
if isinstance(nested_name, str) and nested_name:
return nested_name
name: Final = tool.get("name")
return name if isinstance(name, str) and name else None
def prune_tool_choice_to_surviving_tools(tools: object, tool_choice: object) -> object:
if not isinstance(tool_choice, Mapping):
return tool_choice
if tools is None or isinstance(tools, (str, bytes)) or not isinstance(tools, Sequence):
return tool_choice
surviving: Final = frozenset(tool_name for tool in tools if (tool_name := _tool_definition_name(tool)) is not None)
choice_type: Final = tool_choice.get("type")
if choice_type in ("function", "custom", "tool"):
choice_name: Final = _tool_definition_name(tool_choice)
if choice_name is None or choice_name in surviving:
return tool_choice
if choice_type == "tool":
return {"type": "auto"} if surviving else {"type": "none"}
return "auto" if surviving else "none"
if choice_type == "allowed_tools":
inner: Final = tool_choice.get("allowed_tools")
entries: Final = inner.get("tools") if isinstance(inner, Mapping) else tool_choice.get("tools")
if entries is None or isinstance(entries, (str, bytes)) or not isinstance(entries, Sequence):
return tool_choice
kept: Final = [
entry
for entry in entries
if (entry if isinstance(entry, str) else _tool_definition_name(entry)) in surviving
]
if kept:
if isinstance(inner, Mapping):
nested_inner: Final = dict(inner)
nested_inner["tools"] = kept
nested_choice: Final = dict(tool_choice)
nested_choice["allowed_tools"] = nested_inner
return nested_choice
flat_choice: Final = dict(tool_choice)
flat_choice["tools"] = kept
return flat_choice
return "auto" if surviving else "none"
return tool_choice
class ProxyLogging:
"""
Logging/Custom Handlers for proxy.
@ -3121,6 +3173,8 @@ class ProxyLogging:
if data is not None:
self._process_guardrail_metadata(data)
if "tool_choice" in data:
data["tool_choice"] = prune_tool_choice_to_surviving_tools(data.get("tools"), data["tool_choice"])
return data
except SensitiveDataRouteException as e:

View file

@ -3714,3 +3714,200 @@ async def test_evict_config_param_clears_the_local_layer_and_survives_a_redis_ou
await evict_config_param("router_settings", cache=cache)
assert await cache.in_memory_cache.async_get_cache(_config_cache_key("router_settings")) is None
def _function_tool(name):
return {"type": "function", "function": {"name": name}}
@pytest.mark.parametrize(
"tools,tool_choice,expected",
[
(
[_function_tool("get_time"), _function_tool("get_weather")],
{"type": "function", "function": {"name": "get_weather"}},
{"type": "function", "function": {"name": "get_weather"}},
),
(
[_function_tool("get_time")],
{"type": "function", "function": {"name": "get_weather"}},
"auto",
),
(
[],
{"type": "function", "function": {"name": "get_weather"}},
"none",
),
(
None,
{"type": "function", "function": {"name": "get_weather"}},
{"type": "function", "function": {"name": "get_weather"}},
),
(
[_function_tool("get_time")],
{"type": "custom", "custom": {"name": "get_weather"}},
"auto",
),
(
[_function_tool("get_time"), _function_tool("get_weather")],
{"type": "custom", "custom": {"name": "get_weather"}},
{"type": "custom", "custom": {"name": "get_weather"}},
),
(
[_function_tool("get_time")],
{
"type": "allowed_tools",
"allowed_tools": {
"mode": "auto",
"tools": [_function_tool("get_weather"), _function_tool("get_time")],
},
},
{
"type": "allowed_tools",
"allowed_tools": {"mode": "auto", "tools": [_function_tool("get_time")]},
},
),
(
[_function_tool("get_time")],
{
"type": "allowed_tools",
"allowed_tools": {"mode": "auto", "tools": [_function_tool("get_weather")]},
},
"auto",
),
(
[],
{
"type": "allowed_tools",
"allowed_tools": {"mode": "required", "tools": [_function_tool("get_weather")]},
},
"none",
),
(
[_function_tool("get_time")],
{
"type": "allowed_tools",
"mode": "required",
"tools": [_function_tool("get_weather"), _function_tool("get_time")],
},
{
"type": "allowed_tools",
"mode": "required",
"tools": [_function_tool("get_time")],
},
),
(
[{"name": "get_time", "input_schema": {}}],
{"type": "tool", "name": "get_weather"},
{"type": "auto"},
),
(
[{"name": "get_time", "input_schema": {}}, {"name": "get_weather", "input_schema": {}}],
{"type": "tool", "name": "get_weather"},
{"type": "tool", "name": "get_weather"},
),
([_function_tool("get_time")], "auto", "auto"),
([_function_tool("get_time")], "none", "none"),
([_function_tool("get_time")], {"type": "auto"}, {"type": "auto"}),
([_function_tool("get_time")], {"type": "other"}, {"type": "other"}),
([_function_tool("get_time")], None, None),
(
[_function_tool("get_time")],
{"type": "function", "function": {}},
{"type": "function", "function": {}},
),
],
)
def test_prune_tool_choice_to_surviving_tools(tools, tool_choice, expected):
from litellm.proxy.utils import prune_tool_choice_to_surviving_tools
assert prune_tool_choice_to_surviving_tools(tools, tool_choice) == expected
def test_prune_tool_choice_to_surviving_tools_keeps_surviving_choice_unchanged():
from litellm.proxy.utils import prune_tool_choice_to_surviving_tools
choice = {"type": "function", "function": {"name": "get_weather"}}
tools = [_function_tool("get_time"), _function_tool("get_weather")]
assert prune_tool_choice_to_surviving_tools(tools, choice) is choice
@pytest.mark.asyncio
async def test_pre_call_hook_prunes_tool_choice_after_guardrail_removes_tool():
from litellm.caching.caching import DualCache
from litellm.proxy.utils import ProxyLogging
class RemovingGuardrail(CustomGuardrail):
def __init__(self):
super().__init__(
guardrail_name="tool-pruner",
event_hook=GuardrailEventHooks.pre_call,
default_on=True,
)
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
data["tools"] = [tool for tool in data.get("tools") or [] if tool["function"]["name"] != "get_weather"]
return data
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
original_callbacks = litellm.callbacks.copy() if litellm.callbacks else []
try:
litellm.callbacks = [RemovingGuardrail()]
result = await proxy_logging.pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="<redacted>", user_id="test_user"),
data={
"model": "gpt-5.5",
"messages": [{"role": "user", "content": "hi"}],
"tools": [_function_tool("get_time"), _function_tool("get_weather")],
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
},
call_type="completion",
)
assert result["tools"] == [_function_tool("get_time")]
assert result["tool_choice"] == "auto"
finally:
litellm.callbacks = original_callbacks
@pytest.mark.asyncio
async def test_pre_call_hook_keeps_tool_choice_naming_surviving_tool():
from litellm.caching.caching import DualCache
from litellm.proxy.utils import ProxyLogging
class RemovingGuardrail(CustomGuardrail):
def __init__(self):
super().__init__(
guardrail_name="tool-pruner",
event_hook=GuardrailEventHooks.pre_call,
default_on=True,
)
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
data["tools"] = [tool for tool in data.get("tools") or [] if tool["function"]["name"] != "get_weather"]
return data
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
original_callbacks = litellm.callbacks.copy() if litellm.callbacks else []
try:
litellm.callbacks = [RemovingGuardrail()]
result = await proxy_logging.pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="<redacted>", user_id="test_user"),
data={
"model": "gpt-5.5",
"messages": [{"role": "user", "content": "hi"}],
"tools": [_function_tool("get_time"), _function_tool("get_weather")],
"tool_choice": {"type": "function", "function": {"name": "get_time"}},
},
call_type="completion",
)
assert result["tools"] == [_function_tool("get_time")]
assert result["tool_choice"] == {"type": "function", "function": {"name": "get_time"}}
finally:
litellm.callbacks = original_callbacks