From 64f1ab196bbba49a20562b754c4e9e8619172a8f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:37:41 -0700 Subject: [PATCH] fix(anthropic): keep tool search off for any defer_loading drop path and restore only boolean deferrals A drop path such as tools[1].defer_loading or the SDK's tuple form now turns the tool search mapping off like tools[*].defer_loading, the count path reads the deployment's drop list, and a guardrail write-back restores only a boolean defer_loading --- .../chat/guardrail_translation/handler.py | 8 +++---- .../messages/responses_adapters/handler.py | 13 ++++++------ .../responses_adapters/tool_translation.py | 21 ++++++++++++++++--- .../responses/count_tokens/token_counter.py | 14 +++++++++++-- .../test_anthropic_guardrail_handler.py | 2 ++ .../test_responses_adapters_handler.py | 16 ++++++++++---- .../count_tokens/test_token_counter.py | 3 +++ 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 56a09d655a5..c9c9fcae673 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -259,10 +259,10 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None: - if "defer_loading" in tool: - return {"defer_loading": tool["defer_loading"]} - match tool.get("custom"): - case {"defer_loading": bool() as wrapped}: + match tool: + case {"defer_loading": bool() as deferred}: + return {"defer_loading": deferred} + case {"custom": {"defer_loading": bool() as wrapped}}: return {"custom": {"defer_loading": wrapped}} case _: return None diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 0a6f68f9354..11e0f0fb3fb 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -50,7 +50,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper -from .tool_translation import openai_runs_tool_search +from .tool_translation import maps_openai_tool_search from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] @@ -183,13 +183,12 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) -_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading" - - def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) - return provider == LlmProviders.OPENAI.value and openai_runs_tool_search( - local_model_name(model, provider), forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url") + return provider == LlmProviders.OPENAI.value and maps_openai_tool_search( + local_model_name(model, provider), + forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), + forwarded_kwargs.get("additional_drop_params"), ) @@ -337,7 +336,7 @@ def _build_responses_kwargs( ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), - maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped, + maps_tool_search=_maps_tool_search(model, forwarded_kwargs), ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 4d593a369fa..95ae765452a 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -3,6 +3,8 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from typing import Final, TypeAlias +from pydantic import ConfigDict, TypeAdapter, ValidationError + import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError @@ -13,6 +15,9 @@ from litellm.types.utils import LlmProviders _VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z0-9_]+?)_(?P\d{8})") _UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"}) +_DEFERRAL_DROP_PATH: Final = re.compile(r"tools\[[^\]]*\]\.defer_loading") +_DropPaths: TypeAlias = list[object] | tuple[object, ...] +_DROP_PATHS: Final[TypeAdapter[_DropPaths]] = TypeAdapter(_DropPaths, config=ConfigDict(strict=True)) _TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429" _COMPUTER_MODIFIERS_SINCE: Final = "20250124" _COMPUTER_ZOOM_SINCE: Final = "20251124" @@ -342,7 +347,17 @@ def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError: ) -def openai_runs_tool_search(model: str, api_base: object) -> bool: - return targets_openai_api(api_base) and litellm.supports_tool_search( - model=model, custom_llm_provider=LlmProviders.OPENAI.value +def _drops_deferral(additional_drop_params: object) -> bool: + try: + paths: Final = _DROP_PATHS.validate_python(additional_drop_params) + except ValidationError: + return False + return any(isinstance(path, str) and _DEFERRAL_DROP_PATH.fullmatch(path) is not None for path in paths) + + +def maps_openai_tool_search(model: str, api_base: object, additional_drop_params: object) -> bool: + return ( + targets_openai_api(api_base) + and not _drops_deferral(additional_drop_params) + and litellm.supports_tool_search(model=model, custom_llm_provider=LlmProviders.OPENAI.value) ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 75fe22ee928..16cb4712997 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -18,7 +18,7 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages -from litellm.llms.anthropic.messages.responses_adapters.tool_translation import openai_runs_tool_search +from litellm.llms.anthropic.messages.responses_adapters.tool_translation import maps_openai_tool_search from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -64,6 +64,14 @@ class _CountCredential: credential: str | OpenAIWorkloadIdentityConfig +def _deployment_drop_paths(deployment: Mapping[str, object] | None) -> object: + match deployment: + case {"litellm_params": {"additional_drop_params": object() as additional_drop_params}}: + return additional_drop_params + case _: + return None + + def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError: litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} api_base: Final = litellm_params.get("api_base") @@ -148,6 +156,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat, output_format: JsonValue, api_base: str | None, + additional_drop_params: object, ) -> ResponsesCountRequestParts: match messages_format: case "anthropic": @@ -157,7 +166,7 @@ class OpenAITokenCounter(BaseTokenCounter): system=system, tools=tools, output_format=output_format, - maps_tool_search=openai_runs_tool_search(model_to_use, api_base), + maps_tool_search=maps_openai_tool_search(model_to_use, api_base, additional_drop_params), ) case "openai": input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input( @@ -237,6 +246,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format=messages_format, output_format=output_format, api_base=resolved.api_base, + additional_drop_params=_deployment_drop_paths(deployment), ) # If no input items were produced (e.g., system-only messages), fall back to local counting diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 7457e42fe93..42dbea6c946 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -662,6 +662,7 @@ class TestAnthropicMessagesHandlerInputProcessing: {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False}, {"name": "get_time", "input_schema": {"type": "object", "properties": {}}}, {"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}}, + {"name": "get_news", "input_schema": {"type": "object", "properties": {}}, "defer_loading": "true"}, ], } @@ -673,6 +674,7 @@ class TestAnthropicMessagesHandlerInputProcessing: ("get_weather", False, None), ("get_time", None, None), ("get_quote", None, {"defer_loading": True}), + ("get_news", None, None), ] @pytest.mark.asyncio diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 2ccb0f6ad6f..10ac27a16a5 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,7 @@ import datetime import json import os import sys -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -1025,8 +1025,16 @@ async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_ @pytest.mark.asyncio -async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + "additional_drop_params", + [ + pytest.param(["tools[*].defer_loading"], id="every-tool"), + pytest.param(["tools[1].defer_loading"], id="one-tool-by-index"), + pytest.param(("tools[*].defer_loading",), id="sdk-tuple"), + ], +) +async def test_additional_drop_params_of_a_defer_loading_path_answers_400_for_tool_search_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, additional_drop_params: Sequence[str] ): with pytest.raises(litellm.UnsupportedParamsError) as error: await _openai_request_for( @@ -1034,7 +1042,7 @@ async def test_additional_drop_params_of_every_defer_loading_answers_400_for_too monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], drop_params=False, - additional_drop_params=["tools[*].defer_loading"], + additional_drop_params=additional_drop_params, ) assert "tool_search_tool_bm25_20251119" in error.value.message diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 3de097476c8..dc04a71b2b0 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -274,6 +274,9 @@ async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool( [ pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"), pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"), + pytest.param( + "gpt-5.6-luna", {"additional_drop_params": ["tools[*].defer_loading"]}, id="deployment-drops-deferral" + ), ], ) async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it(