mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(tool_choice): accept allowed_tools in both OpenAI shapes and translate it per endpoint
The chat completions validator now takes the nested chat shape and the flat Responses shape and returns the nested one. The Responses bridge sends the flat shape. Providers other than openai, azure and litellm_proxy raise UnsupportedParamsError, or drop the choice under drop_params.
This commit is contained in:
parent
f23e23cbf8
commit
838e51f903
6 changed files with 252 additions and 102 deletions
|
|
@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
|
|||
import json
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from itertools import accumulate, chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast
|
||||
|
|
@ -17,6 +18,7 @@ from openai.types.responses.response_input_param import (
|
|||
ResponseCustomToolCallOutputParam,
|
||||
ResponseCustomToolCallParam,
|
||||
)
|
||||
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
|
||||
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
|
||||
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
|
|
@ -26,6 +28,10 @@ import litellm
|
|||
from litellm import ModelResponse
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
ALLOWED_TOOL_CHOICE_ADAPTER,
|
||||
to_responses_allowed_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
|
|
@ -732,13 +738,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
def _normalize_tool_choice_for_responses_api(
|
||||
self, tool_choice: _ToolChoiceT
|
||||
) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
|
||||
) -> (
|
||||
_ToolChoiceT
|
||||
| ToolChoiceFunctionParam
|
||||
| ToolChoiceCustomParam
|
||||
| ToolChoiceAllowedParam
|
||||
| Literal["auto", "none", "required"]
|
||||
):
|
||||
"""Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
|
||||
if not isinstance(tool_choice, dict):
|
||||
return tool_choice
|
||||
choice_type: Final = tool_choice.get("type")
|
||||
if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
|
||||
return choice_type
|
||||
if choice_type == "allowed_tools":
|
||||
with suppress(ValidationError):
|
||||
return to_responses_allowed_tool_choice(ALLOWED_TOOL_CHOICE_ADAPTER.validate_python(tool_choice))
|
||||
if choice_type not in ("function", "custom"):
|
||||
return tool_choice
|
||||
if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
|
||||
|
|
|
|||
83
litellm/litellm_core_utils/allowed_tools_tool_choice.py
Normal file
83
litellm/litellm_core_utils/allowed_tools_tool_choice.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam
|
||||
from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam
|
||||
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
PROVIDERS_ACCEPTING_ALLOWED_TOOLS: Final = frozenset({"openai", "azure", "litellm_proxy"})
|
||||
|
||||
|
||||
class _AllowedTools(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
mode: Literal["auto", "required"]
|
||||
tools: tuple[Mapping[str, object], ...]
|
||||
|
||||
|
||||
class _ChatAllowedToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
type: Literal["allowed_tools"]
|
||||
allowed_tools: _AllowedTools
|
||||
|
||||
|
||||
class _ResponsesAllowedToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
type: Literal["allowed_tools"]
|
||||
mode: Literal["auto", "required"]
|
||||
tools: tuple[Mapping[str, object], ...]
|
||||
|
||||
|
||||
AllowedToolChoice: TypeAlias = _ChatAllowedToolChoice | _ResponsesAllowedToolChoice
|
||||
|
||||
ALLOWED_TOOL_CHOICE_ADAPTER: Final[TypeAdapter[AllowedToolChoice]] = TypeAdapter(AllowedToolChoice)
|
||||
|
||||
|
||||
def _allowed_tools(choice: AllowedToolChoice) -> _AllowedTools:
|
||||
match choice:
|
||||
case _ChatAllowedToolChoice(allowed_tools=allowed_tools):
|
||||
return allowed_tools
|
||||
case _ResponsesAllowedToolChoice(mode=mode, tools=tools):
|
||||
return _AllowedTools(mode=mode, tools=tools)
|
||||
|
||||
|
||||
def _chat_tool(tool: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match tool:
|
||||
case {"type": "function", "name": str() as function_name}:
|
||||
return {"type": "function", "function": {"name": function_name}}
|
||||
case {"type": "custom", "name": str() as custom_name}:
|
||||
return {"type": "custom", "custom": {"name": custom_name}}
|
||||
case _:
|
||||
return tool
|
||||
|
||||
|
||||
def _responses_tool(tool: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match tool:
|
||||
case {"type": "function", "function": {"name": str() as function_name}}:
|
||||
return {"type": "function", "name": function_name}
|
||||
case {"type": "custom", "custom": {"name": str() as custom_name}}:
|
||||
return {"type": "custom", "name": custom_name}
|
||||
case _:
|
||||
return tool
|
||||
|
||||
|
||||
def to_chat_allowed_tool_choice(choice: AllowedToolChoice) -> ChatCompletionAllowedToolChoiceParam:
|
||||
allowed_tools: Final = _allowed_tools(choice)
|
||||
return ChatCompletionAllowedToolChoiceParam(
|
||||
type="allowed_tools",
|
||||
allowed_tools=ChatCompletionAllowedToolsParam(
|
||||
mode=allowed_tools.mode, tools=[dict(_chat_tool(tool)) for tool in allowed_tools.tools]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def to_responses_allowed_tool_choice(choice: AllowedToolChoice) -> ToolChoiceAllowedParam:
|
||||
allowed_tools: Final = _allowed_tools(choice)
|
||||
return ToolChoiceAllowedParam(
|
||||
type="allowed_tools",
|
||||
mode=allowed_tools.mode,
|
||||
tools=[dict(_responses_tool(tool)) for tool in allowed_tools.tools],
|
||||
)
|
||||
|
|
@ -82,6 +82,11 @@ from litellm.constants import (
|
|||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
ALLOWED_TOOL_CHOICE_ADAPTER,
|
||||
PROVIDERS_ACCEPTING_ALLOWED_TOOLS,
|
||||
to_chat_allowed_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
bind_budget_reservation_to_callbacks,
|
||||
normalize_drop_params,
|
||||
|
|
@ -5102,6 +5107,17 @@ def get_optional_params(
|
|||
_check_valid_arg(
|
||||
supported_params=supported_params or [],
|
||||
)
|
||||
match non_default_params:
|
||||
case {"tool_choice": {"type": "allowed_tools"}} if custom_llm_provider not in PROVIDERS_ACCEPTING_ALLOWED_TOOLS:
|
||||
if litellm.drop_params is not True and drop_params is not True:
|
||||
raise UnsupportedParamsError(
|
||||
message=f"{custom_llm_provider} does not support an allowed_tools tool_choice, for model={model}. To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
del non_default_params["tool_choice"]
|
||||
case dict():
|
||||
pass
|
||||
## raise exception if provider doesn't support passed in param
|
||||
if custom_llm_provider == "anthropic":
|
||||
## check if unsupported param passed in
|
||||
|
|
@ -9015,14 +9031,15 @@ def validate_chat_completion_tool_choice(
|
|||
if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice:
|
||||
return tool_choice_type
|
||||
|
||||
# OpenAI also allows constraining which tools the model may call:
|
||||
# {"type": "allowed_tools", "allowed_tools": {"mode": ..., "tools": [...]}}.
|
||||
# Pass it through untouched so downstream providers see the full payload.
|
||||
if tool_choice_type == "allowed_tools" and isinstance(
|
||||
tool_choice.get("allowed_tools"), # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # raw dict payload
|
||||
dict,
|
||||
):
|
||||
return tool_choice # pyright: ignore[reportUnknownVariableType] # narrowed raw dict payload
|
||||
if tool_choice_type == "allowed_tools":
|
||||
try:
|
||||
return dict(to_chat_allowed_tool_choice(ALLOWED_TOOL_CHOICE_ADAPTER.validate_python(tool_choice)))
|
||||
except ValidationError as exc:
|
||||
raise BadRequestError(
|
||||
message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec. {exc}",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
) from exc
|
||||
|
||||
# Standard OpenAI format: {"type": "function", "function": {...}}
|
||||
if tool_choice.get("type") is None or tool_choice.get("function") is None:
|
||||
|
|
|
|||
|
|
@ -4015,6 +4015,32 @@ def test_output_item_done_with_stream_map_keeps_empty_delta():
|
|||
({"type": "required"}, "required"),
|
||||
("auto", "auto"),
|
||||
({"type": "function", "function": {"name": "get_weather"}}, {"type": "function", "name": "get_weather"}),
|
||||
(
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
{"type": "custom", "custom": {"name": "apply_patch"}},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "name": "get_weather"},
|
||||
{"type": "custom", "name": "apply_patch"},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire(
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ from litellm.utils import (
|
|||
calculate_max_parallel_requests,
|
||||
client,
|
||||
get_non_default_completion_params,
|
||||
get_optional_params,
|
||||
get_optional_params_image_gen,
|
||||
get_prompt_cache_min_tokens,
|
||||
get_supported_openai_params,
|
||||
|
|
@ -8265,6 +8266,107 @@ def test_validate_tool_choice_without_model_is_still_a_400():
|
|||
assert exc_info.value.model == ""
|
||||
|
||||
|
||||
CHAT_ALLOWED_TOOLS: Final = {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_weather"}},
|
||||
{"type": "custom", "custom": {"name": "apply_patch"}},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
CHAT_ALLOWED_TOOLS,
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "name": "get_weather"},
|
||||
{"type": "custom", "name": "apply_patch"},
|
||||
{"type": "mcp", "server_label": "deepwiki"},
|
||||
],
|
||||
},
|
||||
],
|
||||
ids=["chat_nested", "responses_flat"],
|
||||
)
|
||||
def test_validate_tool_choice_allowed_tools_takes_the_chat_shape(tool_choice):
|
||||
"""Both OpenAI allowed_tools shapes are accepted and come out in the chat completions shape."""
|
||||
assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) == CHAT_ALLOWED_TOOLS
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
{"type": "allowed_tools"},
|
||||
{"type": "allowed_tools", "mode": "required"},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "sometimes", "tools": []}},
|
||||
{"type": "allowed_tools", "allowed_tools": "get_weather", "mode": "auto"},
|
||||
],
|
||||
)
|
||||
def test_validate_tool_choice_malformed_allowed_tools_is_a_400(tool_choice):
|
||||
with pytest.raises(litellm.BadRequestError, match="Invalid tool choice") as exc_info:
|
||||
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
ALLOWED_TOOLS_REQUEST_TOOLS: Final = [
|
||||
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(("model", "custom_llm_provider"), [("gpt-5.5", "openai"), ("gpt-5.5", "azure")])
|
||||
def test_get_optional_params_keeps_allowed_tools_for_openai_shaped_providers(model, custom_llm_provider):
|
||||
optional_params = get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=CHAT_ALLOWED_TOOLS,
|
||||
)
|
||||
assert optional_params["tool_choice"] == CHAT_ALLOWED_TOOLS
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider"),
|
||||
[("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), ("deepseek-chat", "deepseek")],
|
||||
)
|
||||
def test_get_optional_params_rejects_allowed_tools_a_provider_cannot_take(model, custom_llm_provider):
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="does not support an allowed_tools tool_choice") as exc_info:
|
||||
get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=CHAT_ALLOWED_TOOLS,
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider"),
|
||||
[("claude-haiku-4-5", "anthropic"), ("gemini-2.5-flash", "gemini"), ("deepseek-chat", "deepseek")],
|
||||
)
|
||||
def test_get_optional_params_drops_allowed_tools_under_drop_params(model, custom_llm_provider):
|
||||
optional_params = get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=ALLOWED_TOOLS_REQUEST_TOOLS,
|
||||
tool_choice=CHAT_ALLOWED_TOOLS,
|
||||
drop_params=True,
|
||||
)
|
||||
assert "tool_choice" not in optional_params
|
||||
assert optional_params["tools"]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def _vcr_outcome_gate_local_testing(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
import re
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.utils import validate_chat_completion_tool_choice
|
||||
|
||||
MODEL: Final = "anthropic/claude-haiku-4-5"
|
||||
|
||||
|
||||
def test_validate_tool_choice_none():
|
||||
"""Test that None is returned as-is."""
|
||||
result = validate_chat_completion_tool_choice(None, model=MODEL)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_validate_tool_choice_string():
|
||||
"""Test that string values are returned as-is."""
|
||||
assert validate_chat_completion_tool_choice("auto", model=MODEL) == "auto"
|
||||
assert validate_chat_completion_tool_choice("none", model=MODEL) == "none"
|
||||
assert validate_chat_completion_tool_choice("required", model=MODEL) == "required"
|
||||
|
||||
|
||||
def test_validate_tool_choice_standard_dict():
|
||||
"""Test standard OpenAI format with function."""
|
||||
tool_choice = {"type": "function", "function": {"name": "my_function"}}
|
||||
result = validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert result == tool_choice
|
||||
|
||||
|
||||
def test_validate_tool_choice_cursor_format():
|
||||
"""Cursor IDE format {"type": "auto"} is unwrapped to the bare string."""
|
||||
assert validate_chat_completion_tool_choice({"type": "auto"}, model=MODEL) == "auto"
|
||||
assert validate_chat_completion_tool_choice({"type": "none"}, model=MODEL) == "none"
|
||||
assert validate_chat_completion_tool_choice({"type": "required"}, model=MODEL) == "required"
|
||||
|
||||
|
||||
def test_validate_tool_choice_allowed_tools_passthrough():
|
||||
"""Chat completions accepts the allowed_tools shape and keeps its payload intact."""
|
||||
tool_choice = {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "function": {"name": "get_time"}}],
|
||||
},
|
||||
}
|
||||
assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice
|
||||
|
||||
|
||||
def test_validate_tool_choice_allowed_tools_without_payload_is_a_400():
|
||||
"""allowed_tools without its nested object stays a caller error."""
|
||||
tool_choice = {"type": "allowed_tools"}
|
||||
with pytest.raises(litellm.BadRequestError, match="Invalid tool choice"):
|
||||
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
{},
|
||||
{"type": "invalid"},
|
||||
{"type": "function"},
|
||||
{"name": "lookup_fruit"},
|
||||
{"type": "file_search"},
|
||||
],
|
||||
)
|
||||
def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice):
|
||||
"""A dict shape chat completions cannot carry is the caller's mistake: a 400 that names the field, never a 500."""
|
||||
with pytest.raises(
|
||||
litellm.BadRequestError, match=f"Invalid tool choice, tool_choice={re.escape(str(tool_choice))}\\. Please ensure"
|
||||
) as exc_info:
|
||||
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.model == MODEL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", [123, []])
|
||||
def test_validate_tool_choice_invalid_type_is_a_400(tool_choice):
|
||||
"""A non-str, non-dict tool_choice is rejected as a 400 that names the type it got."""
|
||||
with pytest.raises(
|
||||
litellm.BadRequestError, match=f"Got={re.escape(str(type(tool_choice)))}\\. Expecting str, or dict\\."
|
||||
) as exc_info:
|
||||
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
def test_validate_tool_choice_without_model_is_still_a_400():
|
||||
"""Callers that predate the model argument keep getting a 400, with an empty model on the error."""
|
||||
with pytest.raises(litellm.BadRequestError, match="Invalid tool choice") as exc_info:
|
||||
validate_chat_completion_tool_choice({"type": "bogus"})
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.model == ""
|
||||
Loading…
Add table
Reference in a new issue