fix(tool-choice): keep the 400 when LiteLLM runs the tools, report auto after a drop

Backport of #50467 to lts/1.108.x.
Cherry-picked from 5f8f54a6ca (litellm_allowed_tools_flat_tool_choice). Conflicts resolved by applying this commit's hunks to the line's simpler Responses bridge handler and responses() bridge branch, and appending its new tests to the line's test files.
This commit is contained in:
mateo-berri 2026-11-06 19:32:06 -08:00
parent 5b05913f16
commit c1ac5fafd0
11 changed files with 397 additions and 7 deletions

View file

@ -124,6 +124,28 @@ def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsTool
return _chat_layout(parsed)
def unenforced_allowed_tools_tool_choice(tool_choice: object, custom_llm_provider: str | None) -> bool:
return is_allowed_tools_tool_choice(tool_choice) and (
custom_llm_provider not in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS
or parse_allowed_tools_tool_choice(tool_choice) is None
)
def refuse_unenforced_allowed_tools_tool_choice(
tool_choice: object, custom_llm_provider: str | None, model: str
) -> None:
if unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider):
raise BadRequestError(
message=(
f"This allowed_tools tool_choice can't be enforced on custom_llm_provider={custom_llm_provider}, and "
"LiteLLM runs the tool calls on this request, so drop_params does not drop it. Send only the allowed "
"tools in `tools` with tool_choice='auto' or 'required'."
),
model=model,
llm_provider=custom_llm_provider or "",
)
def chat_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ChatCompletionAllowedToolChoiceParam:
return ChatCompletionAllowedToolChoiceParam(
type="allowed_tools",

View file

@ -71,6 +71,7 @@ from litellm.exceptions import LiteLLMUnknownProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
drops_unsupported_allowed_tools_tool_choice,
refuse_unenforced_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
from litellm.litellm_core_utils.audio_utils.utils import (
@ -5359,7 +5360,8 @@ def completion(
tool_choice = validate_chat_completion_tool_choice(
tool_choice=tool_choice,
model=model,
drop_params=drops_unsupported,
drop_params=drops_unsupported
and not may_name_litellm_mcp_gateway(tools), # pyright: ignore[reportUnknownArgumentType] # untyped tools list
)
# validate optional params
stop = validate_openai_optional_params(stop=stop)
@ -5617,6 +5619,15 @@ def completion(
if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
tools = _drop_anthropic_only_tool_keys(tools=tools)
if skip_mcp_handler:
refuse_unenforced_allowed_tools_tool_choice(
tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
object, tool_choice
),
custom_llm_provider=custom_llm_provider,
model=model,
)
drops_tool_choice: Final = drops_unsupported_allowed_tools_tool_choice(
tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
object, tool_choice

View file

@ -8,6 +8,7 @@ from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.allowed_tools_tool_choice import unenforced_allowed_tools_tool_choice
from litellm.responses.additional_tools import hoist_additional_tools
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
@ -26,6 +27,17 @@ from litellm.types.utils import ModelResponse
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
def _with_the_tool_choice_sent(
responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None
) -> ResponsesAPIOptionalRequestParams:
tool_choice: Final[object] = responses_api_request.get( # pyright: ignore[reportUnknownMemberType] # ToolChoice has an untyped member
"tool_choice"
)
if not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider):
return responses_api_request
return {**responses_api_request, "tool_choice": None}
class LiteLLMCompletionTransformationHandler:
def response_api_handler(
self,
@ -58,12 +70,13 @@ class LiteLLMCompletionTransformationHandler:
**kwargs,
)
)
echoed_request: Final = _with_the_tool_choice_sent(bridged_request, custom_llm_provider)
if _is_async:
return self.async_response_api_handler(
litellm_completion_request=litellm_completion_request,
request_input=bridged_input,
responses_api_request=bridged_request,
responses_api_request=echoed_request,
**kwargs,
)
@ -81,7 +94,7 @@ class LiteLLMCompletionTransformationHandler:
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
chat_completion_response=litellm_completion_response,
request_input=bridged_input,
responses_api_request=bridged_request,
responses_api_request=echoed_request,
)
)
@ -92,7 +105,7 @@ class LiteLLMCompletionTransformationHandler:
model=model,
litellm_custom_stream_wrapper=litellm_completion_response,
request_input=bridged_input,
responses_api_request=bridged_request,
responses_api_request=echoed_request,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
)

View file

@ -26,6 +26,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
configured_injection_points,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import refuse_unenforced_allowed_tools_tool_choice
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -463,6 +464,13 @@ def _bridge_kwargs(
return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys})
def _refuse_a_choice_the_mcp_tool_run_would_ignore(
skip_mcp_handler: bool, tool_choice: object, custom_llm_provider: str | None, model: str
) -> None:
if skip_mcp_handler:
refuse_unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider, model)
_ResponsesCompatibilityFailure: TypeAlias = Literal["encrypted_task_unsupported"]
@ -1498,6 +1506,12 @@ def responses(
if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api):
bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params)
_refuse_a_choice_the_mcp_tool_run_would_ignore(
skip_mcp_handler, # pyright: ignore[reportUnknownArgumentType] # popped from untyped kwargs
tool_choice,
custom_llm_provider,
model,
)
return litellm_completion_transformation_handler.response_api_handler(
model=model,
input=input,

View file

@ -6,6 +6,10 @@ from typing import Final, TypeAlias, cast
from typing_extensions import TypedDict, Unpack
from litellm.constants import DEFAULT_TOOL_LOOP_MAX_ROUNDS
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
is_allowed_tools_tool_choice,
refuse_unenforced_allowed_tools_tool_choice,
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -32,11 +36,32 @@ class ToolLoopMaxRoundsExceeded(RuntimeError):
super().__init__(f"model still requested tool calls on round {max_rounds} of {max_rounds}")
def _validate_tool_loop_args(max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None:
def _text(value: object) -> str | None:
return value if isinstance(value, str) else None
def _refuse_unenforced_tool_choice(model: str, completion_kwargs: _ToolLoopCompletionKwargs) -> None:
import litellm
tool_choice: Final = completion_kwargs.get("tool_choice")
if is_allowed_tools_tool_choice(tool_choice):
refuse_unenforced_allowed_tools_tool_choice(
tool_choice,
litellm.get_llm_provider(
model=model,
custom_llm_provider=_text(completion_kwargs.get("custom_llm_provider")),
api_base=_text(completion_kwargs.get("api_base")),
)[1],
model,
)
def _validate_tool_loop_args(model: str, max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None:
if max_rounds < 1:
raise ValueError(f"max_rounds must be >= 1, got {max_rounds}")
if completion_kwargs.get("stream"):
raise ValueError("run_tool_loop requires whole responses; stream=True is not supported")
_refuse_unenforced_tool_choice(model, completion_kwargs)
def _expect_model_response(response: object) -> ModelResponse:
@ -146,7 +171,7 @@ def run_tool_loop(
"""
import litellm
_validate_tool_loop_args(max_rounds, completion_kwargs)
_validate_tool_loop_args(model, max_rounds, completion_kwargs)
history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns
for round_number in range(1, max_rounds + 1):
message = (
@ -197,7 +222,7 @@ async def arun_tool_loop(
"""
import litellm
_validate_tool_loop_args(max_rounds, completion_kwargs)
_validate_tool_loop_args(model, max_rounds, completion_kwargs)
history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns
for round_number in range(1, max_rounds + 1):
message = (

View file

@ -4,11 +4,14 @@ from typing import Final
import pytest
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
ChatAllowedToolsToolChoice,
is_allowed_tools_tool_choice,
chat_allowed_tools_tool_choice,
parse_allowed_tools_tool_choice,
refuse_unenforced_allowed_tools_tool_choice,
unenforced_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
@ -145,3 +148,39 @@ def test_parse_rejects_a_malformed_choice_in_either_layout(tool_choice: dict[str
)
def test_is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None, expected: bool):
assert is_allowed_tools_tool_choice(tool_choice) is expected
_FLAT_CHOICE: Final = {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}
@pytest.mark.parametrize(
("tool_choice", "custom_llm_provider"),
[
(_FLAT_CHOICE, "anthropic"),
(_CHAT_LAYOUT, "hosted_vllm"),
({"type": "allowed_tools", "mode": "required", "tools": []}, "openai"),
({"type": "allowed_tools", "tools": [{"type": "function", "name": "get_time"}]}, "azure"),
],
)
def test_a_tool_runner_refuses_a_choice_nothing_would_enforce(tool_choice: object, custom_llm_provider: str):
with pytest.raises(BadRequestError, match="drop_params does not drop it") as exc:
refuse_unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider, "the-model")
assert exc.value.status_code == 400
@pytest.mark.parametrize(
("tool_choice", "custom_llm_provider"),
[
(_FLAT_CHOICE, "openai"),
(_CHAT_LAYOUT, "azure"),
(_FLAT_CHOICE, "litellm_proxy"),
("required", "anthropic"),
({"type": "function", "function": {"name": "get_time"}}, "anthropic"),
(None, "anthropic"),
],
)
def test_a_tool_runner_keeps_a_choice_the_provider_enforces_or_any_other_choice(
tool_choice: object, custom_llm_provider: str
):
assert not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider)

View file

@ -540,3 +540,116 @@ def test_streamed_responses_over_anthropic_emit_the_responses_event_sequence(res
assert deltas == "Argentina won."
assert completed.response.status == "completed"
assert completed.response.output[0].content[0].text == "Argentina won."
_FLAT_ALLOWED_WEATHER: Final = {
"type": "allowed_tools",
"mode": "required",
"tools": [{"type": "function", "name": "get_weather"}],
}
_RESPONSES_TOOLS: Final = (
{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}},
{"type": "function", "name": "get_time", "parameters": {"type": "object", "properties": {}}},
)
def _chat_answer() -> httpx.Response:
return httpx.Response(
200,
json={
"id": "chatcmpl-echo",
"object": "chat.completion",
"created": 1739462947,
"model": "the-model",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
@pytest.mark.usefixtures(httpx_transport.__name__)
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.asyncio
async def test_responses_reports_auto_for_an_allowed_tools_choice_it_dropped(
respx_mock: respx.MockRouter, use_async: bool
) -> None:
route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock(
return_value=_anthropic_message({"type": "text", "text": "done"})
)
call: Final = {
"model": "anthropic/claude-haiku-4-5",
"input": "What is the weather?",
"tools": list(_RESPONSES_TOOLS),
"tool_choice": _FLAT_ALLOWED_WEATHER,
"drop_params": True,
"api_key": "sk-ant-fake",
}
response: Final = await litellm.aresponses(**call) if use_async else litellm.responses(**call)
assert "tool_choice" not in json.loads(route.calls[0].request.content)
assert response.tool_choice == "auto"
def test_streamed_responses_report_auto_for_an_allowed_tools_choice_it_dropped(respx_mock: respx.MockRouter) -> None:
respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock(
return_value=httpx.Response(
200,
headers={"content-type": "text/event-stream"},
content=_anthropic_sse(
{
"type": "message_start",
"message": {
"id": "msg_dropped_choice",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [],
"usage": {"input_tokens": 9, "output_tokens": 1},
},
},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "done"}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}},
{"type": "message_stop"},
),
)
)
events: Final = tuple(
litellm.responses(
model="anthropic/claude-haiku-4-5",
input="What is the weather?",
tools=list(_RESPONSES_TOOLS),
tool_choice=_FLAT_ALLOWED_WEATHER,
drop_params=True,
stream=True,
api_key="sk-ant-fake",
)
)
reported: Final = tuple(
event.response.tool_choice
for event in events
if event.type in (ResponsesAPIStreamEvents.RESPONSE_CREATED, ResponsesAPIStreamEvents.RESPONSE_COMPLETED)
)
assert reported == ("auto", "auto")
def test_responses_through_chat_keeps_reporting_a_choice_the_provider_received(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post("http://localhost:12346/v1/chat/completions").mock(return_value=_chat_answer())
response: Final = litellm.responses(
model="openai/gpt-5.5",
input="What is the weather?",
tools=list(_RESPONSES_TOOLS),
tool_choice=_FLAT_ALLOWED_WEATHER,
use_chat_completions_api=True,
api_base="http://localhost:12346/v1",
api_key="sk-fake",
)
assert json.loads(route.calls[0].request.content)["tool_choice"]["type"] == "allowed_tools"
assert response.tool_choice["type"] == "allowed_tools"
assert list(response.tool_choice["tools"]) == _FLAT_ALLOWED_WEATHER["tools"]

View file

@ -1486,3 +1486,27 @@ async def test_streaming_mcp_event_order_and_response_id_consistency(
assert lifecycle_ids[0] == lifecycle_ids[1]
assert lifecycle_ids[2] != lifecycle_ids[0]
assert not [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR]
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_mcp_tools_keep_the_400_for_a_choice_the_chat_bridge_would_drop(
respx_mock: respx.MockRouter, stream: bool
) -> None:
async def call_with_the_choice() -> None:
response: Final = await litellm.aresponses(
model="anthropic/claude-haiku-4-5",
input="Call echo with PING",
tools=[dict(_ECHO_MCP_TOOL)],
tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "echo"}]},
drop_params=True,
stream=stream,
api_key="sk-ant-fake",
)
if stream:
[event async for event in response]
with _echo_tool_served("PING"), pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"):
await call_with_the_choice()
assert not respx_mock.calls

View file

@ -1479,3 +1479,41 @@ async def test_request_selected_mcp_guardrail_blocks_before_upstream(monkeypatch
assert upstream.await_count == (0 if selected else 1)
tool_message = model_call.await_args.kwargs["messages"][-1]
assert ("request-selected MCP block" in tool_message["content"]) is selected
_MCP_ECHO_TOOL: Final = MCPTool.model_validate(
{"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}},
by_name=False,
)
_FLAT_ALLOWED_ECHO: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "echo"}]}
@pytest.mark.asyncio
@pytest.mark.parametrize("require_approval", ["never", "always"])
@pytest.mark.parametrize("stream", [False, True])
async def test_mcp_tools_keep_the_400_for_an_allowed_tools_choice_the_provider_cannot_enforce(
respx_mock: respx.MockRouter, require_approval: str, stream: bool
):
execute: Final = AsyncMock(return_value=[])
with (
patch.object(
LiteLLM_Proxy_MCP_Handler,
"process_mcp_tools_without_openai_transform",
AsyncMock(return_value=([_MCP_ECHO_TOOL], {"echo": "demo"})),
),
patch.object(LiteLLM_Proxy_MCP_Handler, "execute_tool_calls", execute),
pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"),
):
await litellm.acompletion(
model="anthropic/claude-haiku-4-5",
messages=[{"role": "user", "content": "Call echo"}],
tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": require_approval}],
tool_choice=_FLAT_ALLOWED_ECHO,
drop_params=True,
stream=stream,
api_key="sk-ant-fake",
)
execute.assert_not_awaited()
assert not respx_mock.calls

View file

@ -516,6 +516,23 @@ def test_completion_drop_params_drops_an_allowed_tools_tool_choice_it_cannot_sen
assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_time", "get_weather"]
@pytest.mark.asyncio
async def test_a_malformed_allowed_tools_tool_choice_stays_a_400_on_mcp_gateway_tools(respx_mock: respx.MockRouter):
with pytest.raises(litellm.BadRequestError, match="Invalid tool choice") as exc:
await litellm.acompletion(
model="openai/gpt-5.5",
messages=[{"role": "user", "content": "What time is it?"}],
tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}],
tool_choice={"type": "allowed_tools", "mode": "required", "tools": []},
drop_params=True,
api_base="http://localhost:12346/v1",
api_key="fake_api_key",
)
assert exc.value.status_code == 400
assert not respx_mock.calls
_CLAUDE_TOOL_SEARCH_CALL: Final = {
"id": "srvtoolu_01search",
"type": "function",

View file

@ -248,6 +248,80 @@ def test_stream_rejected_before_any_request(respx_mock: respx.MockRouter) -> Non
assert route.call_count == 0
_FLAT_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "a"}]}
_EMPTY_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": []}
@pytest.mark.parametrize(
("model", "tool_choice"),
[("anthropic/claude-haiku-4-5", _FLAT_ALLOWED_TOOLS), ("openai/gpt-5-mini", _EMPTY_ALLOWED_TOOLS)],
)
def test_allowed_tools_choice_nothing_enforces_is_refused_under_drop_params(
respx_mock: respx.MockRouter, model: str, tool_choice: dict[str, object]
) -> None:
executed: Final[list[ChatCompletionMessageToolCall]] = []
def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage:
executed.append(tool_call)
return _tool_result(tool_call)
with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"):
litellm.run_tool_loop(
model=model,
messages=[{"role": "user", "content": "hi"}],
tools=WEATHER_TOOLS,
execute_tool=execute,
tool_choice=tool_choice,
drop_params=True,
api_key="sk-test",
)
assert not executed
assert not respx_mock.calls
@pytest.mark.asyncio
async def test_async_loop_refuses_an_allowed_tools_choice_the_provider_cannot_enforce(
respx_mock: respx.MockRouter,
) -> None:
async def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage:
return _tool_result(tool_call)
with pytest.raises(litellm.BadRequestError, match="drop_params does not drop it"):
await litellm.arun_tool_loop(
model="anthropic/claude-haiku-4-5",
messages=[{"role": "user", "content": "hi"}],
tools=WEATHER_TOOLS,
execute_tool=execute,
tool_choice=_FLAT_ALLOWED_TOOLS,
drop_params=True,
api_key="sk-test",
)
assert not respx_mock.calls
def test_allowed_tools_choice_the_provider_enforces_reaches_it_every_round(respx_mock: respx.MockRouter) -> None:
respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock(
side_effect=[
httpx.Response(200, json=_openai_response(None, [_tool_call("call_1", "get_weather", {"city": "Paris"})])),
httpx.Response(200, json=_openai_response("done")),
]
)
answer: Final = litellm.run_tool_loop(
model="openai/gpt-5-mini",
messages=[{"role": "user", "content": "hi"}],
tools=WEATHER_TOOLS,
execute_tool=_tool_result,
tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]},
api_key="sk-test",
)
assert answer == "done"
assert [body["tool_choice"]["type"] for body in _request_bodies(respx_mock)] == ["allowed_tools", "allowed_tools"]
def test_custom_tool_call_raises_type_error_without_executing(respx_mock: respx.MockRouter) -> None:
custom_response: Final = _openai_response(
None,