From e12ed84cdfb0825130c60d921e23d59b63448cb3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 07:11:46 +0000 Subject: [PATCH] fix(websearch): drop forced tool_choice from /v1/messages interception follow-up The Anthropic /v1/messages websearch-interception follow-up call inherited the caller's forced tool_choice, which the deployment hook had repointed at litellm_web_search. The follow-up model was therefore obliged to call the search tool again and that second tool_use shipped to a client that never declared the tool. The chat-completions and Responses paths already strip tool_choice; the Anthropic one only removed max_tokens. Stripping it from the request patch alone is insufficient: each executor merges the base optional params (which still carry the repointed choice) back in via dict.update, which can only overwrite keys, not delete them. So the executors now drop tool_choice when the patch does not carry it, mirroring the chat-completions loop. tools stays so the model can still be offered the tool, matching the sibling paths. --- .../websearch_interception/handler.py | 10 ++- litellm/llms/custom_httpx/llm_http_handler.py | 7 +- .../test_websearch_interception_handler.py | 78 ++++++++++++++++++- .../test_websearch_streaming_wrap.py | 56 ++++++++++++- 4 files changed, 144 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 21d990e8e60..86c2965e2ac 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -1110,6 +1110,8 @@ class WebSearchInterceptionLogger(CustomLogger): optional_params = dict(anthropic_messages_optional_request_params) optional_params.update(request_patch.optional_params) + if "tool_choice" not in request_patch.optional_params: + optional_params.pop("tool_choice", None) max_tokens = request_patch.max_tokens if max_tokens is None: max_tokens = cast(Optional[int], optional_params.pop("max_tokens", None)) @@ -1213,8 +1215,10 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug(f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request") - optional_params_without_max_tokens = { - k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens" + followup_optional_params = { + k: v + for k, v in anthropic_messages_optional_request_params.items() + if k not in {"max_tokens", "tool_choice"} } kwargs_for_followup = self._prepare_followup_kwargs(kwargs) @@ -1232,7 +1236,7 @@ class WebSearchInterceptionLogger(CustomLogger): model=full_model_name, messages=follow_up_messages, max_tokens=max_tokens, - optional_params=optional_params_without_max_tokens, + optional_params=followup_optional_params, kwargs=kwargs_for_followup, ) return patch, structured_results diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..bb0e929b561 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5098,6 +5098,8 @@ class BaseLLMHTTPHandler: optional_params.update(patch.optional_params) if patch.tools is not None: optional_params["tools"] = patch.tools + if "tool_choice" not in patch.optional_params: + optional_params.pop("tool_choice", None) max_tokens = patch.max_tokens if max_tokens is None: @@ -5170,10 +5172,13 @@ class BaseLLMHTTPHandler: optional_params.update(patch.optional_params) if patch.tools is not None: optional_params["tools"] = patch.tools + drop_tool_choice = "tool_choice" not in patch.optional_params optional_params = { k: v for k, v in optional_params.items() - if k != "stream" and k != "_code_interpreter_interception_converted_stream" + if k != "stream" + and k != "_code_interpreter_interception_converted_stream" + and not (drop_tool_choice and k == "tool_choice") } internal_keys = {"litellm_logging_obj"} diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index b6ff3b70a4d..0c0300b1033 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -4,7 +4,7 @@ Unit tests for WebSearch Interception Handler Tests the WebSearchInterceptionLogger class and helper functions. """ -from unittest.mock import AsyncMock, MagicMock, Mock +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -152,6 +152,82 @@ async def test_async_build_agentic_loop_plan_returns_request_patch(): assert plan.request_patch.kwargs["temperature"] == 0.2 +@pytest.mark.asyncio +async def test_build_anthropic_request_patch_drops_forced_tool_choice(): + """Regression (#35334): the follow-up /v1/messages patch must not carry a + forced tool_choice. The deployment hook repoints a caller's forced + tool_choice at litellm_web_search; if it survives into the follow-up call + the model is obliged to call the search tool again and that tool_use leaks + to a client that never declared it. tools is kept, matching the + chat-completions and Responses paths.""" + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + logger._execute_search = AsyncMock(return_value=("results", None)) # type: ignore + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + patch_obj, _ = await logger._build_anthropic_request_patch( + model="bedrock/claude-3-5-sonnet", + messages=[{"role": "user", "content": "search litellm"}], + tool_calls=[ + {"id": "toolu_1", "type": "tool_use", "name": "litellm_web_search", "input": {"query": "litellm"}} + ], + thinking_blocks=[], + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "tools": [{"name": "litellm_web_search", "input_schema": {}}], + "tool_choice": {"type": "tool", "name": "litellm_web_search"}, + }, + logging_obj=logging_obj, + kwargs={}, + ) + + assert "tool_choice" not in patch_obj.optional_params + assert patch_obj.optional_params["tools"] == [{"name": "litellm_web_search", "input_schema": {}}] + + +@pytest.mark.asyncio +async def test_execute_agentic_loop_does_not_force_tool_choice_on_followup(): + """Regression (#35334): even though the base optional params still carry a + forced tool_choice (repointed to litellm_web_search), the follow-up + /v1/messages call must not receive it. Dropping it only from the request + patch is insufficient because the executor merges the base params back in, + so the executor must strip it too. tools stays available.""" + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + logger._execute_search = AsyncMock(return_value=("results", None)) # type: ignore + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + captured: dict = {} + + async def fake_acreate(**kwargs): + captured.update(kwargs) + return {"id": "msg", "content": [{"type": "text", "text": "answer"}]} + + with patch( + "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", + new=AsyncMock(side_effect=fake_acreate), + ): + await logger._execute_agentic_loop( + model="bedrock/claude-3-5-sonnet", + messages=[{"role": "user", "content": "search litellm"}], + tool_calls=[ + {"id": "toolu_1", "type": "tool_use", "name": "litellm_web_search", "input": {"query": "litellm"}} + ], + thinking_blocks=[], + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "tools": [{"name": "litellm_web_search", "input_schema": {}}], + "tool_choice": {"type": "tool", "name": "litellm_web_search"}, + }, + logging_obj=logging_obj, + stream=False, + kwargs={}, + ) + + assert "tool_choice" not in captured + assert captured["tools"] == [{"name": "litellm_web_search", "input_schema": {}}] + + @pytest.mark.asyncio async def test_internal_flags_filtered_from_followup_kwargs(): """Test that internal _websearch_interception flags are filtered from follow-up request kwargs. diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_streaming_wrap.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_streaming_wrap.py index f221e07a57d..8ac6097bbdf 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_streaming_wrap.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_streaming_wrap.py @@ -7,7 +7,7 @@ FakeAnthropicMessagesStreamIterator when the original request was streaming but converted to non-streaming for WebSearch interception. """ -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -16,7 +16,10 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) -from litellm.types.integrations.custom_logger import AgenticLoopPlan +from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, +) def _anthropic_response() -> dict: @@ -219,6 +222,55 @@ class TestCallAgenticCompletionHooksWrapping: assert isinstance(result, FakeAnthropicMessagesStreamIterator) + @pytest.mark.asyncio + async def test_anthropic_plan_execution_drops_forced_tool_choice(self): + """Regression (#35334): the plan-based follow-up /v1/messages call must + not inherit the caller's forced tool_choice (repointed to + litellm_web_search). The base optional params still carry it, so the + executor must strip it even though the patch already dropped it. tools + stays available.""" + captured: dict = {} + + async def fake_acreate(**kwargs): + captured.update(kwargs) + return _anthropic_response() + + plan = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model="bedrock/claude-3-5-sonnet", + messages=[{"role": "user", "content": "results"}], + max_tokens=1024, + optional_params={"temperature": 0.2}, + ), + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + with patch( + "litellm.anthropic_interface.messages.acreate", + new=AsyncMock(side_effect=fake_acreate), + ): + await BaseLLMHTTPHandler()._execute_anthropic_agentic_plan( + plan=plan, + model="bedrock/claude-3-5-sonnet", + messages=[{"role": "user", "content": "search"}], + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "tools": [{"name": "litellm_web_search", "input_schema": {}}], + "tool_choice": {"type": "tool", "name": "litellm_web_search"}, + }, + logging_obj=logging_obj, + kwargs={}, + depth=0, + max_loops=3, + fingerprints=[], + fingerprint="fp", + ) + + assert "tool_choice" not in captured + assert captured["tools"] == [{"name": "litellm_web_search", "input_schema": {}}] + @pytest.mark.asyncio async def test_tail_path_wraps_when_no_loop_runs(self): plan = AgenticLoopPlan(run_agentic_loop=False)