mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(tool_choice): accept the Responses API's flat allowed_tools layout on chat completions
An allowed_tools tool_choice in the flat Responses layout ({"type": "allowed_tools", "mode": ..., "tools": [...]})
answered 400 on litellm.completion, acompletion, Router and the proxy's chat and /v1/responses-to-chat routes. One
parser now reads both layouts into the nested Chat layout, and the existing Responses bridge flattens it back for
/v1/responses, so either layout reaches OpenAI in the shape each endpoint expects
This commit is contained in:
parent
5a392b9e5f
commit
e56c060363
8 changed files with 184 additions and 40 deletions
|
|
@ -36,7 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
|
|||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
parse_chat_allowed_tools_tool_choice,
|
||||
parse_allowed_tools_tool_choice,
|
||||
responses_allowed_tools_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
|
|
@ -802,7 +802,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
|
||||
return choice_type
|
||||
if choice_type == "allowed_tools":
|
||||
parsed: Final = parse_chat_allowed_tools_tool_choice(
|
||||
parsed: Final = parse_allowed_tools_tool_choice(
|
||||
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
|
||||
object, tool_choice
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final, Literal
|
||||
|
||||
from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam
|
||||
from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam
|
||||
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -11,6 +13,9 @@ ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "lite
|
|||
CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
|
||||
'{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
|
||||
)
|
||||
RESPONSES_ALLOWED_TOOLS_LAYOUT: Final = (
|
||||
'{"type": "allowed_tools", "mode": "auto" | "required", "tools": [<at least one tool>]}'
|
||||
)
|
||||
|
||||
|
||||
class ChatAllowedTools(BaseModel):
|
||||
|
|
@ -27,7 +32,17 @@ class ChatAllowedToolsToolChoice(BaseModel):
|
|||
allowed_tools: ChatAllowedTools
|
||||
|
||||
|
||||
_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
|
||||
class ResponsesAllowedToolsToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
type: Literal["allowed_tools"]
|
||||
mode: Literal["auto", "required"]
|
||||
tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
|
||||
|
||||
|
||||
_ALLOWED_TOOLS_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice]] = (
|
||||
TypeAdapter(ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice)
|
||||
)
|
||||
|
||||
|
||||
_SELECTOR: Final = TypeAdapter(dict[str, object])
|
||||
|
|
@ -70,18 +85,51 @@ def raise_if_allowed_tools_tool_choice_unsupported(
|
|||
)
|
||||
|
||||
|
||||
def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
|
||||
try:
|
||||
return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _has_name(selector: Mapping[str, object]) -> bool:
|
||||
name: Final = selector.get("name")
|
||||
return isinstance(name, str) and name != ""
|
||||
|
||||
|
||||
def _chat_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
tool_type: Final = tool.get("type")
|
||||
if not isinstance(tool_type, str) or tool_type not in ("function", "custom") or tool_type in tool:
|
||||
return tool
|
||||
if not _has_name(tool):
|
||||
return None
|
||||
return {**{key: value for key, value in tool.items() if key != "name"}, tool_type: {"name": tool["name"]}}
|
||||
|
||||
|
||||
def _chat_layout(choice: ResponsesAllowedToolsToolChoice) -> ChatAllowedToolsToolChoice | None:
|
||||
nested: Final = tuple(_chat_allowed_tool(tool) for tool in choice.tools)
|
||||
if any(tool is None for tool in nested):
|
||||
return None
|
||||
return ChatAllowedToolsToolChoice(
|
||||
type="allowed_tools",
|
||||
allowed_tools=ChatAllowedTools(mode=choice.mode, tools=tuple(tool for tool in nested if tool is not None)),
|
||||
)
|
||||
|
||||
|
||||
def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
|
||||
try:
|
||||
parsed: Final = _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
|
||||
except ValidationError:
|
||||
return None
|
||||
match parsed:
|
||||
case ChatAllowedToolsToolChoice():
|
||||
return parsed
|
||||
case ResponsesAllowedToolsToolChoice():
|
||||
return _chat_layout(parsed)
|
||||
|
||||
|
||||
def chat_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ChatCompletionAllowedToolChoiceParam:
|
||||
return ChatCompletionAllowedToolChoiceParam(
|
||||
type="allowed_tools",
|
||||
allowed_tools=ChatCompletionAllowedToolsParam(
|
||||
mode=choice.allowed_tools.mode, tools=[dict(tool) for tool in choice.allowed_tools.tools]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
tool_type: Final = tool.get("type")
|
||||
if tool_type not in ("function", "custom"):
|
||||
|
|
|
|||
|
|
@ -86,7 +86,9 @@ from litellm.constants import (
|
|||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
CHAT_ALLOWED_TOOLS_LAYOUT,
|
||||
parse_chat_allowed_tools_tool_choice,
|
||||
RESPONSES_ALLOWED_TOOLS_LAYOUT,
|
||||
chat_allowed_tools_tool_choice,
|
||||
parse_allowed_tools_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
bind_budget_reservation_to_callbacks,
|
||||
|
|
@ -9665,28 +9667,25 @@ def validate_chat_completion_tool_choice(
|
|||
tool_choice_type = tool_choice.get("type")
|
||||
if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice:
|
||||
return tool_choice_type
|
||||
if (
|
||||
tool_choice_type == "allowed_tools"
|
||||
and (
|
||||
parse_chat_allowed_tools_tool_choice(
|
||||
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
|
||||
object, tool_choice
|
||||
)
|
||||
if tool_choice_type == "allowed_tools":
|
||||
allowed_tools: Final = parse_allowed_tools_tool_choice(
|
||||
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
|
||||
object, tool_choice
|
||||
)
|
||||
is None
|
||||
)
|
||||
):
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Invalid tool choice, tool_choice={tool_choice}. On chat completions an allowed_tools "
|
||||
f"tool_choice must be {CHAT_ALLOWED_TOOLS_LAYOUT}"
|
||||
),
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
if allowed_tools is None:
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Invalid tool choice, tool_choice={tool_choice}. An allowed_tools tool_choice must be "
|
||||
f"{CHAT_ALLOWED_TOOLS_LAYOUT} or {RESPONSES_ALLOWED_TOOLS_LAYOUT}"
|
||||
),
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
return dict(chat_allowed_tools_tool_choice(allowed_tools))
|
||||
|
||||
# Standard OpenAI format: {"type": "function", "function": {...}}
|
||||
if tool_choice_type != "allowed_tools" and (tool_choice_type is None or tool_choice.get("function") is None):
|
||||
if tool_choice_type is None or tool_choice.get("function") is None:
|
||||
raise BadRequestError(
|
||||
message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec",
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -4558,6 +4558,10 @@ def test_output_item_done_with_stream_map_keeps_empty_delta():
|
|||
},
|
||||
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
),
|
||||
(
|
||||
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire(
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ import pytest
|
|||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
ChatAllowedToolsToolChoice,
|
||||
is_allowed_tools_tool_choice,
|
||||
parse_chat_allowed_tools_tool_choice,
|
||||
chat_allowed_tools_tool_choice,
|
||||
parse_allowed_tools_tool_choice,
|
||||
responses_allowed_tools_tool_choice,
|
||||
)
|
||||
|
||||
|
|
@ -26,7 +27,7 @@ _CHAT_LAYOUT: Final[dict[str, object]] = {
|
|||
|
||||
|
||||
def _parsed(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice:
|
||||
parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice)
|
||||
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
|
||||
assert parsed is not None
|
||||
return parsed
|
||||
|
||||
|
|
@ -72,18 +73,59 @@ def test_responses_layout_refuses_to_rewrite_a_malformed_or_conflicting_entry(to
|
|||
assert responses_allowed_tools_tool_choice(_parsed(tool_choice)) is None
|
||||
|
||||
|
||||
_RESPONSES_LAYOUT: Final[dict[str, object]] = {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "name": "get_time"},
|
||||
{"type": "custom", "name": "apply_patch", "x-vendor": 1},
|
||||
{"type": "mcp", "server_label": "docs"},
|
||||
{"type": "function", "function": {"name": "already_nested"}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_responses_layout_is_read_into_the_chat_layout():
|
||||
assert chat_allowed_tools_tool_choice(_parsed(_RESPONSES_LAYOUT)) == {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "required",
|
||||
"tools": [
|
||||
{"type": "function", "function": {"name": "get_time"}},
|
||||
{"type": "custom", "x-vendor": 1, "custom": {"name": "apply_patch"}},
|
||||
{"type": "mcp", "server_label": "docs"},
|
||||
{"type": "function", "function": {"name": "already_nested"}},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_responses_layout_round_trips_back_to_what_the_caller_sent():
|
||||
flat_only: Final = {**_RESPONSES_LAYOUT, "tools": list(_RESPONSES_LAYOUT["tools"])[:3]}
|
||||
assert responses_allowed_tools_tool_choice(_parsed(flat_only)) == flat_only
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "any", "tools": [{"type": "function"}]}},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}, "extra": 1},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": []},
|
||||
{"type": "allowed_tools", "mode": "none", "tools": [{"type": "function", "name": "get_time"}]},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function"}]},
|
||||
{"type": "allowed_tools", "mode": "auto", "tools": [{"type": "custom", "name": ""}]},
|
||||
{
|
||||
"type": "allowed_tools",
|
||||
"mode": "auto",
|
||||
"tools": [{"type": "function", "name": "get_time"}],
|
||||
"allowed_tools": {"mode": "auto", "tools": [{"type": "function", "name": "get_time"}]},
|
||||
},
|
||||
{"type": "function", "function": {"name": "get_time"}},
|
||||
],
|
||||
)
|
||||
def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, object]):
|
||||
assert parse_chat_allowed_tools_tool_choice(tool_choice) is None
|
||||
def test_parse_rejects_a_malformed_choice_in_either_layout(tool_choice: dict[str, object]):
|
||||
assert parse_allowed_tools_tool_choice(tool_choice) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -305,6 +305,44 @@ class TestUseResponsesApiBridgeFlag:
|
|||
assert request_body["messages"] == [{"role": "user", "content": "Hello"}]
|
||||
assert response.output[0].content[0].text == "Answer"
|
||||
|
||||
def test_bridge_sends_a_responses_layout_allowed_tools_tool_choice_in_the_chat_layout(
|
||||
self, respx_mock: respx.MockRouter
|
||||
):
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "my-custom-model",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
litellm.responses(
|
||||
model="openai/my-custom-model",
|
||||
input="What time is it?",
|
||||
use_chat_completions_api=True,
|
||||
tools=[
|
||||
{"type": "function", "name": name, "parameters": {"type": "object", "properties": {}}}
|
||||
for name in ("get_time", "get_weather")
|
||||
],
|
||||
tool_choice={"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]},
|
||||
api_key="fake-provider-api-key",
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert json.loads(upstream.calls[0].request.read())["tool_choice"] == {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]},
|
||||
}
|
||||
|
||||
def test_bridge_merges_instructions_and_developer_input_for_databricks(self, respx_mock: respx.MockRouter):
|
||||
upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
|
|
|
|||
|
|
@ -557,15 +557,21 @@ _ALLOWED_TOOLS_TOOL_CHOICE: Final = {
|
|||
"type": "allowed_tools",
|
||||
"allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]},
|
||||
}
|
||||
_RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE: Final = {
|
||||
"type": "allowed_tools",
|
||||
"mode": "required",
|
||||
"tools": [{"type": "function", "name": "get_time"}],
|
||||
}
|
||||
_TWO_TOOLS: Final = [
|
||||
{"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}},
|
||||
{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", [_ALLOWED_TOOLS_TOOL_CHOICE, _RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE])
|
||||
@pytest.mark.parametrize("model", ["openai/gpt-5.5", "azure/my-gpt-deployment"])
|
||||
def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat(
|
||||
respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str
|
||||
respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str, tool_choice: dict[str, object]
|
||||
):
|
||||
mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock(
|
||||
return_value=httpx.Response(status_code=200, json=openai_api_response)
|
||||
|
|
@ -575,7 +581,7 @@ def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat
|
|||
model=model,
|
||||
messages=[{"role": "user", "content": "What time is it?"}],
|
||||
tools=_TWO_TOOLS,
|
||||
tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE,
|
||||
tool_choice=tool_choice,
|
||||
api_base="http://localhost:12346/v1",
|
||||
api_key="fake_api_key",
|
||||
api_version="2025-04-01-preview",
|
||||
|
|
|
|||
|
|
@ -8715,22 +8715,29 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice):
|
|||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize("mode", ["auto", "required"])
|
||||
def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode: str):
|
||||
tool_choice: Final = {
|
||||
def test_validate_tool_choice_allowed_tools_either_layout_comes_back_in_the_chat_layout(mode: str):
|
||||
chat_layout: Final = {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": mode,
|
||||
"tools": [{"type": "function", "function": {"name": "get_time"}}, {"type": "mcp", "server_label": "docs"}],
|
||||
},
|
||||
}
|
||||
assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice
|
||||
responses_layout: Final = {
|
||||
"type": "allowed_tools",
|
||||
"mode": mode,
|
||||
"tools": [{"type": "function", "name": "get_time"}, {"type": "mcp", "server_label": "docs"}],
|
||||
}
|
||||
assert validate_chat_completion_tool_choice(chat_layout, model=MODEL) == chat_layout
|
||||
assert validate_chat_completion_tool_choice(responses_layout, model=MODEL) == chat_layout
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]},
|
||||
{"type": "allowed_tools", "mode": "required", "tools": [{"type": "function"}]},
|
||||
{"type": "allowed_tools", "mode": "required", "tools": []},
|
||||
{"type": "allowed_tools"},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "required"}},
|
||||
{"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": []}},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue