diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 81cf383b73d..51f84860c3e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -28,7 +28,7 @@ from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - parse_chat_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, ) from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params @@ -737,7 +737,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type if choice_type == "allowed_tools": - parsed: Final = parse_chat_allowed_tools_tool_choice( + parsed: Final = parse_allowed_tools_tool_choice( cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic object, tool_choice ) diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 5c8f5d83ece..55656977fa7 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,6 +1,8 @@ from collections.abc import Mapping from typing import Final, Literal +from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam +from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError @@ -11,6 +13,9 @@ ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "lite CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' ) +RESPONSES_ALLOWED_TOOLS_LAYOUT: Final = ( + '{"type": "allowed_tools", "mode": "auto" | "required", "tools": []}' +) class ChatAllowedTools(BaseModel): @@ -27,7 +32,17 @@ class ChatAllowedToolsToolChoice(BaseModel): allowed_tools: ChatAllowedTools -_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice) +class ResponsesAllowedToolsToolChoice(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: Literal["allowed_tools"] + mode: Literal["auto", "required"] + tools: tuple[Mapping[str, object], ...] = Field(min_length=1) + + +_ALLOWED_TOOLS_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice]] = ( + TypeAdapter(ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice) +) _SELECTOR: Final = TypeAdapter(dict[str, object]) @@ -70,18 +85,51 @@ def raise_if_allowed_tools_tool_choice_unsupported( ) -def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: - try: - return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) - except ValidationError: - return None - - def _has_name(selector: Mapping[str, object]) -> bool: name: Final = selector.get("name") return isinstance(name, str) and name != "" +def _chat_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None: + tool_type: Final = tool.get("type") + if not isinstance(tool_type, str) or tool_type not in ("function", "custom") or tool_type in tool: + return tool + if not _has_name(tool): + return None + return {**{key: value for key, value in tool.items() if key != "name"}, tool_type: {"name": tool["name"]}} + + +def _chat_layout(choice: ResponsesAllowedToolsToolChoice) -> ChatAllowedToolsToolChoice | None: + nested: Final = tuple(_chat_allowed_tool(tool) for tool in choice.tools) + if any(tool is None for tool in nested): + return None + return ChatAllowedToolsToolChoice( + type="allowed_tools", + allowed_tools=ChatAllowedTools(mode=choice.mode, tools=tuple(tool for tool in nested if tool is not None)), + ) + + +def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: + try: + parsed: Final = _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) + except ValidationError: + return None + match parsed: + case ChatAllowedToolsToolChoice(): + return parsed + case ResponsesAllowedToolsToolChoice(): + return _chat_layout(parsed) + + +def chat_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ChatCompletionAllowedToolChoiceParam: + return ChatCompletionAllowedToolChoiceParam( + type="allowed_tools", + allowed_tools=ChatCompletionAllowedToolsParam( + mode=choice.allowed_tools.mode, tools=[dict(tool) for tool in choice.allowed_tools.tools] + ), + ) + + def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None: tool_type: Final = tool.get("type") if tool_type not in ("function", "custom"): diff --git a/litellm/utils.py b/litellm/utils.py index 424c353b41d..bced83cae12 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -82,7 +82,9 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.allowed_tools_tool_choice import ( CHAT_ALLOWED_TOOLS_LAYOUT, - parse_chat_allowed_tools_tool_choice, + RESPONSES_ALLOWED_TOOLS_LAYOUT, + chat_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice, ) from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, @@ -8843,28 +8845,25 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type - if ( - tool_choice_type == "allowed_tools" - and ( - parse_chat_allowed_tools_tool_choice( - cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic - object, tool_choice - ) + if tool_choice_type == "allowed_tools": + allowed_tools: Final = parse_allowed_tools_tool_choice( + cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice ) - is None - ) - ): - raise BadRequestError( - message=( - f"Invalid tool choice, tool_choice={tool_choice}. On chat completions an allowed_tools " - f"tool_choice must be {CHAT_ALLOWED_TOOLS_LAYOUT}" - ), - model=model, - llm_provider="", ) + if allowed_tools is None: + raise BadRequestError( + message=( + f"Invalid tool choice, tool_choice={tool_choice}. An allowed_tools tool_choice must be " + f"{CHAT_ALLOWED_TOOLS_LAYOUT} or {RESPONSES_ALLOWED_TOOLS_LAYOUT}" + ), + model=model, + llm_provider="", + ) + return dict(chat_allowed_tools_tool_choice(allowed_tools)) # Standard OpenAI format: {"type": "function", "function": {...}} - if tool_choice_type != "allowed_tools" and (tool_choice_type is None or tool_choice.get("function") is None): + if tool_choice_type is None or tool_choice.get("function") is None: raise BadRequestError( message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec", model=model, diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 44821c72690..95037761b11 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3993,6 +3993,10 @@ def test_output_item_done_with_stream_map_keeps_empty_delta(): }, {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, ), + ( + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, + ), ], ) async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire( diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index 6ebbdcf73bf..2353308ab98 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -7,7 +7,8 @@ import pytest from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ChatAllowedToolsToolChoice, is_allowed_tools_tool_choice, - parse_chat_allowed_tools_tool_choice, + chat_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, ) @@ -26,7 +27,7 @@ _CHAT_LAYOUT: Final[dict[str, object]] = { def _parsed(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice: - parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) assert parsed is not None return parsed @@ -72,18 +73,59 @@ def test_responses_layout_refuses_to_rewrite_a_malformed_or_conflicting_entry(to assert responses_allowed_tools_tool_choice(_parsed(tool_choice)) is None +_RESPONSES_LAYOUT: Final[dict[str, object]] = { + "type": "allowed_tools", + "mode": "required", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "apply_patch", "x-vendor": 1}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "function": {"name": "already_nested"}}, + ], +} + + +def test_responses_layout_is_read_into_the_chat_layout(): + assert chat_allowed_tools_tool_choice(_parsed(_RESPONSES_LAYOUT)) == { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "x-vendor": 1, "custom": {"name": "apply_patch"}}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "function": {"name": "already_nested"}}, + ], + }, + } + + +def test_responses_layout_round_trips_back_to_what_the_caller_sent(): + flat_only: Final = {**_RESPONSES_LAYOUT, "tools": list(_RESPONSES_LAYOUT["tools"])[:3]} + assert responses_allowed_tools_tool_choice(_parsed(flat_only)) == flat_only + + @pytest.mark.parametrize( "tool_choice", [ - {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}, {"type": "allowed_tools", "allowed_tools": {"mode": "any", "tools": [{"type": "function"}]}}, {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}, "extra": 1}, + {"type": "allowed_tools", "mode": "auto", "tools": []}, + {"type": "allowed_tools", "mode": "none", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "custom", "name": ""}]}, + { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_time"}], + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + }, {"type": "function", "function": {"name": "get_time"}}, ], ) -def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, object]): - assert parse_chat_allowed_tools_tool_choice(tool_choice) is None +def test_parse_rejects_a_malformed_choice_in_either_layout(tool_choice: dict[str, object]): + assert parse_allowed_tools_tool_choice(tool_choice) is None @pytest.mark.parametrize( diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 699144a31bb..2e5a6860e5c 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -322,6 +322,44 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" + def test_bridge_sends_a_responses_layout_allowed_tools_tool_choice_in_the_chat_layout( + self, respx_mock: respx.MockRouter + ): + upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.responses( + model="openai/my-custom-model", + input="What time is it?", + use_chat_completions_api=True, + tools=[ + {"type": "function", "name": name, "parameters": {"type": "object", "properties": {}}} + for name in ("get_time", "get_weather") + ], + tool_choice={"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, + api_key="fake-provider-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read())["tool_choice"] == { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + } + def test_bridge_merges_instructions_and_developer_input_for_databricks(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( return_value=httpx.Response( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 54bfcb237c6..91902863785 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -426,15 +426,21 @@ _ALLOWED_TOOLS_TOOL_CHOICE: Final = { "type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, } +_RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE: Final = { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "get_time"}], +} _TWO_TOOLS: Final = [ {"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}}, {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}, ] +@pytest.mark.parametrize("tool_choice", [_ALLOWED_TOOLS_TOOL_CHOICE, _RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE]) @pytest.mark.parametrize("model", ["openai/gpt-5.5", "azure/my-gpt-deployment"]) def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat( - respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str, tool_choice: dict[str, object] ): mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock( return_value=httpx.Response(status_code=200, json=openai_api_response) @@ -444,7 +450,7 @@ def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat model=model, messages=[{"role": "user", "content": "What time is it?"}], tools=_TWO_TOOLS, - tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, + tool_choice=tool_choice, api_base="http://localhost:12346/v1", api_key="fake_api_key", api_version="2025-04-01-preview", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 94dc5ffb1bc..ff7c3bc188b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -7448,22 +7448,29 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("mode", ["auto", "required"]) -def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode: str): - tool_choice: Final = { +def test_validate_tool_choice_allowed_tools_either_layout_comes_back_in_the_chat_layout(mode: str): + chat_layout: Final = { "type": "allowed_tools", "allowed_tools": { "mode": mode, "tools": [{"type": "function", "function": {"name": "get_time"}}, {"type": "mcp", "server_label": "docs"}], }, } - assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice + responses_layout: Final = { + "type": "allowed_tools", + "mode": mode, + "tools": [{"type": "function", "name": "get_time"}, {"type": "mcp", "server_label": "docs"}], + } + assert validate_chat_completion_tool_choice(chat_layout, model=MODEL) == chat_layout + assert validate_chat_completion_tool_choice(responses_layout, model=MODEL) == chat_layout @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "tool_choice", [ - {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function"}]}, + {"type": "allowed_tools", "mode": "required", "tools": []}, {"type": "allowed_tools"}, {"type": "allowed_tools", "allowed_tools": {"mode": "required"}}, {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": []}},