diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 5c8f5d83ece..cdc24481680 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -8,6 +8,23 @@ from litellm.exceptions import BadRequestError ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) +HOSTED_TOOL_CHOICE_TYPES: Final = frozenset( + { + "file_search", + "web_search", + "web_search_preview", + "web_search_preview_2025_03_11", + "computer", + "computer_use", + "computer_use_preview", + "code_interpreter", + "image_generation", + "mcp", + "shell", + "apply_patch", + } +) + CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' ) @@ -70,6 +87,37 @@ def raise_if_allowed_tools_tool_choice_unsupported( ) +def hosted_tool_choice_type(tool_choice: object) -> str | None: + """The hosted tool a Responses API ``tool_choice`` forces, such as ``web_search``, or ``None`` for any other choice.""" + try: + choice_type: Final = _TOOL_CHOICE_TYPE.validate_python(tool_choice).type + except ValidationError: + return None + return choice_type if choice_type in HOSTED_TOOL_CHOICE_TYPES else None + + +def raise_if_hosted_tool_choice_unsupported( + tool_choice: object, + custom_llm_provider: str, + model: str, +) -> None: + """Anthropic drops a tool_choice dict it does not recognise and Bedrock sends a tool named "", so a forced hosted + tool stops here with a 400 on providers that cannot run it. ``drop_params`` is deliberately not consulted: the + request required the tool, and dropping the choice would let the model answer without it.""" + hosted: Final = hosted_tool_choice_type(tool_choice) + if hosted is None or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: + return + raise BadRequestError( + message=( + f"tool_choice type '{hosted}' forces an OpenAI hosted tool and is only supported for " + f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. " + "drop_params does not drop it, since that would let the model answer without the tool the request required." + ), + model=model, + llm_provider=custom_llm_provider, + ) + + def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: try: return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) diff --git a/litellm/main.py b/litellm/main.py index f3620d7a8c2..febf9503a7e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -76,6 +76,7 @@ from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.allowed_tools_tool_choice import ( raise_if_allowed_tools_tool_choice_unsupported, + raise_if_hosted_tool_choice_unsupported, ) from litellm.litellm_core_utils.asyncify import asyncify, run_async_function from litellm.litellm_core_utils.audio_utils.utils import ( @@ -5915,6 +5916,13 @@ def completion( custom_llm_provider=custom_llm_provider, model=model, ) + raise_if_hosted_tool_choice_unsupported( + tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ), + custom_llm_provider=custom_llm_provider, + model=model, + ) if provider_specific_header is not None: headers.update( diff --git a/litellm/utils.py b/litellm/utils.py index b9bf872e853..d9ab3626859 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -86,6 +86,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.allowed_tools_tool_choice import ( CHAT_ALLOWED_TOOLS_LAYOUT, + HOSTED_TOOL_CHOICE_TYPES, parse_chat_allowed_tools_tool_choice, ) from litellm.litellm_core_utils.core_helpers import ( @@ -9757,7 +9758,11 @@ def validate_chat_completion_tool_choice( ) # Standard OpenAI format: {"type": "function", "function": {...}} - if tool_choice_type != "allowed_tools" and (tool_choice_type is None or tool_choice.get("function") is None): + if ( + tool_choice_type != "allowed_tools" + and tool_choice_type not in HOSTED_TOOL_CHOICE_TYPES + and (tool_choice_type is None or tool_choice.get("function") is None) + ): raise BadRequestError( message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec", model=model, diff --git a/tests/integration/providers/test_hosted_tool_choice_wire.py b/tests/integration/providers/test_hosted_tool_choice_wire.py new file mode 100644 index 00000000000..8060b8f9fdd --- /dev/null +++ b/tests/integration/providers/test_hosted_tool_choice_wire.py @@ -0,0 +1,60 @@ +"""A chat completions ``tool_choice`` that forces an OpenAI hosted tool, such as ``{"type": "web_search"}``, reaches the +Responses API unchanged, and is a 400 before any provider call where the provider cannot enforce it.""" + +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, list_value +from integration._support.openai_wire import responses_reply +from integration._support.provider import SharedProvider +from pydantic import JsonValue + +_RESPONSES_MODEL: Final = "openai/responses/gpt-5.6-sol" +_ANTHROPIC_MODEL: Final = "anthropic/claude-haiku-4-5" +_WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def _choice_id(tool_choice: dict[str, JsonValue]) -> str: + return str(tool_choice["type"]) + + +def _chat_request(model: str, tool_choice: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model": model, + "messages": [{"role": "user", "content": "What is the weather in Paris right now?"}], + "tools": [_WEATHER_TOOL, {"type": tool_choice["type"]}], + "tool_choice": tool_choice, + } + + +@pytest.mark.parametrize( + "tool_choice", + [{"type": "web_search"}, {"type": "web_search_preview"}, {"type": "file_search"}], + ids=_choice_id, +) +def test_hosted_tool_choice_reaches_the_responses_api_unchanged( + gateway: Gateway, provider: SharedProvider, tool_choice: dict[str, JsonValue] +) -> None: + provider.expect(responses_reply("resp_hosted_tool_choice", "gpt-5.6-sol", "It is 18C in Paris.", stream=False)) + response: Final = gateway.request("POST", "/v1/chat/completions", _chat_request(_RESPONSES_MODEL, tool_choice)) + assert response.status_code == 200, response.text + assert "It is 18C in Paris." in response.text, response.text + received: Final = provider.received() + assert [(request.method, request.target) for request in received] == [("POST", "/responses")] + sent: Final = JSON_OBJECT.validate_json(received[0].body) + assert sent["tool_choice"] == tool_choice + assert {"type": tool_choice["type"]} in list_value(sent["tools"]), sent["tools"] + + +def test_hosted_tool_choice_is_a_400_before_any_call_where_the_provider_cannot_enforce_it( + gateway: Gateway, provider: SharedProvider +) -> None: + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_request(_ANTHROPIC_MODEL, {"type": "web_search"}) + ) + assert response.status_code == 400, response.text + assert "web_search" in response.text and "only supported for" in response.text, response.text + assert provider.received() == () diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index 6ebbdcf73bf..d020c7fcd85 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -4,10 +4,13 @@ from typing import Final import pytest +from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ChatAllowedToolsToolChoice, + hosted_tool_choice_type, is_allowed_tools_tool_choice, parse_chat_allowed_tools_tool_choice, + raise_if_hosted_tool_choice_unsupported, responses_allowed_tools_tool_choice, ) @@ -98,3 +101,42 @@ def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, objec ) def test_is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None, expected: bool): assert is_allowed_tools_tool_choice(tool_choice) is expected + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + ({"type": "web_search"}, "web_search"), + ({"type": "mcp", "server_label": "docs"}, "mcp"), + ({"type": "function", "function": {"name": "get_time"}}, None), + ({"type": "allowed_tools"}, None), + ("required", None), + (None, None), + ], +) +def test_hosted_tool_choice_type(tool_choice: Mapping[str, object] | str | None, expected: str | None): + assert hosted_tool_choice_type(tool_choice) == expected + + +@pytest.mark.parametrize("provider", ["openai", "azure", "litellm_proxy"]) +def test_hosted_tool_choice_passes_where_the_responses_api_runs_the_tool(provider: str): + assert ( + raise_if_hosted_tool_choice_unsupported({"type": "web_search"}, custom_llm_provider=provider, model="m") is None + ) + + +@pytest.mark.parametrize("provider", ["anthropic", "bedrock", "gemini", "vertex_ai"]) +def test_hosted_tool_choice_is_a_400_where_the_provider_cannot_run_it(provider: str): + with pytest.raises(BadRequestError, match=r"tool_choice type 'file_search'.*only supported for") as exc: + raise_if_hosted_tool_choice_unsupported({"type": "file_search"}, custom_llm_provider=provider, model="m") + assert exc.value.status_code == 400 + assert f"custom_llm_provider={provider}" in str(exc.value) + + +def test_a_function_tool_choice_is_not_a_hosted_tool_anywhere(): + assert ( + raise_if_hosted_tool_choice_unsupported( + {"type": "function", "function": {"name": "get_time"}}, custom_llm_provider="anthropic", model="m" + ) + is None + ) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 548f78d148b..0509b8dcf30 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -610,6 +610,69 @@ def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot assert not respx_mock.calls +_HOSTED_TOOL_CHOICE: Final = {"type": "web_search"} +_HOSTED_TOOLS: Final[list[dict[str, object]]] = [*_TWO_TOOLS, {"type": "web_search"}] + + +def test_completion_forwards_a_hosted_tool_choice_to_the_responses_api_unchanged(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + json={ + "id": "resp_hosted", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_hosted", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "18C and foggy", "annotations": []}], + } + ], + "usage": {"input_tokens": 40, "output_tokens": 5, "total_tokens": 45}, + } + ) + + response: Final = litellm.completion( + model="openai/responses/gpt-5.6", + api_key="sk-test", + api_base="https://api.openai.com/v1", + messages=[{"role": "user", "content": "What is the weather in Paris right now?"}], + tools=_HOSTED_TOOLS, + tool_choice=_HOSTED_TOOL_CHOICE, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["tool_choice"] == _HOSTED_TOOL_CHOICE + assert {"type": "web_search"} in sent["tools"] + assert response.choices[0].message.content == "18C and foggy" + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("model", ["anthropic/claude-sonnet-4-5", "gemini/gemini-2.5-flash", "bedrock/converse/x"]) +def test_completion_hosted_tool_choice_is_a_400_where_the_provider_cannot_run_it( + respx_mock: respx.MockRouter, model: str, drop_params: bool +): + with pytest.raises( + litellm.BadRequestError, match=r"tool_choice type 'web_search'.*only supported for azure, litellm_proxy, openai" + ) as exc: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris right now?"}], + tools=_TWO_TOOLS, + tool_choice=_HOSTED_TOOL_CHOICE, + drop_params=drop_params, + api_key="fake_api_key", + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + assert exc.value.status_code == 400 + assert not respx_mock.calls + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", @@ -6390,7 +6453,9 @@ def test_aiohttp_openai_warns_only_when_http2_enabled( assert warned is http2_on -@pytest.mark.parametrize("tool_choice", [{"type": "bogus"}, {"name": "lookup_fruit"}, {"type": "file_search"}]) +@pytest.mark.parametrize( + "tool_choice", [{"type": "bogus"}, {"name": "lookup_fruit"}, {"type": "function", "name": "lookup_fruit"}] +) def test_completion_rejects_untranslatable_tool_choice_with_a_400(tool_choice): with pytest.raises(litellm.BadRequestError) as exc_info: litellm.completion( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 44bdd187da4..cf073ad21e5 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8809,7 +8809,7 @@ def test_validate_tool_choice_cursor_format(): {"type": "invalid"}, {"type": "function"}, {"name": "lookup_fruit"}, - {"type": "file_search"}, + {"type": "web_search_tool"}, ], ) def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): @@ -8823,6 +8823,23 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): assert exc_info.value.model == MODEL +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "web_search"}, + {"type": "web_search_preview"}, + {"type": "file_search"}, + {"type": "code_interpreter"}, + {"type": "mcp", "server_label": "docs", "name": "search"}, + ], + ids=lambda choice: str(choice["type"]), +) +def test_validate_tool_choice_forcing_a_hosted_tool_passes_through(tool_choice: dict[str, object]): + """Forcing an OpenAI hosted tool is a Responses API choice the bridge sends as is, so validation leaves it alone.""" + assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("mode", ["auto", "required"]) def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode: str):