fix(anthropic): keep tool search off for any defer_loading drop path and restore only boolean deferrals

A drop path such as tools[1].defer_loading or the SDK's tuple form now turns the tool search mapping off like tools[*].defer_loading, the count path reads the deployment's drop list, and a guardrail write-back restores only a boolean defer_loading
This commit is contained in:
mateo-berri 2026-10-31 00:37:41 -07:00
parent 06dcd798cf
commit 64f1ab196b
7 changed files with 57 additions and 20 deletions

View file

@ -259,10 +259,10 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s
def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None:
if "defer_loading" in tool:
return {"defer_loading": tool["defer_loading"]}
match tool.get("custom"):
case {"defer_loading": bool() as wrapped}:
match tool:
case {"defer_loading": bool() as deferred}:
return {"defer_loading": deferred}
case {"custom": {"defer_loading": bool() as wrapped}}:
return {"custom": {"defer_loading": wrapped}}
case _:
return None

View file

@ -50,7 +50,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
from .tool_translation import openai_runs_tool_search
from .tool_translation import maps_openai_tool_search
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
@ -183,13 +183,12 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading"
def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider"))
return provider == LlmProviders.OPENAI.value and openai_runs_tool_search(
local_model_name(model, provider), forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url")
return provider == LlmProviders.OPENAI.value and maps_openai_tool_search(
local_model_name(model, provider),
forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
forwarded_kwargs.get("additional_drop_params"),
)
@ -337,7 +336,7 @@ def _build_responses_kwargs(
),
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")),
maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped,
maps_tool_search=_maps_tool_search(model, forwarded_kwargs),
)
# Normalize reasoning effort based on model capabilities

View file

@ -3,6 +3,8 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, replace
from typing import Final, TypeAlias
from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
@ -13,6 +15,9 @@ from litellm.types.utils import LlmProviders
_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P<family>[a-z0-9_]+?)_(?P<version>\d{8})")
_UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"})
_DEFERRAL_DROP_PATH: Final = re.compile(r"tools\[[^\]]*\]\.defer_loading")
_DropPaths: TypeAlias = list[object] | tuple[object, ...]
_DROP_PATHS: Final[TypeAdapter[_DropPaths]] = TypeAdapter(_DropPaths, config=ConfigDict(strict=True))
_TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429"
_COMPUTER_MODIFIERS_SINCE: Final = "20250124"
_COMPUTER_ZOOM_SINCE: Final = "20251124"
@ -342,7 +347,17 @@ def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError:
)
def openai_runs_tool_search(model: str, api_base: object) -> bool:
return targets_openai_api(api_base) and litellm.supports_tool_search(
model=model, custom_llm_provider=LlmProviders.OPENAI.value
def _drops_deferral(additional_drop_params: object) -> bool:
try:
paths: Final = _DROP_PATHS.validate_python(additional_drop_params)
except ValidationError:
return False
return any(isinstance(path, str) and _DEFERRAL_DROP_PATH.fullmatch(path) is not None for path in paths)
def maps_openai_tool_search(model: str, api_base: object, additional_drop_params: object) -> bool:
return (
targets_openai_api(api_base)
and not _drops_deferral(additional_drop_params)
and litellm.supports_tool_search(model=model, custom_llm_provider=LlmProviders.OPENAI.value)
)

View file

@ -18,7 +18,7 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
from litellm.llms.anthropic.messages.responses_adapters.tool_translation import openai_runs_tool_search
from litellm.llms.anthropic.messages.responses_adapters.tool_translation import maps_openai_tool_search
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
@ -64,6 +64,14 @@ class _CountCredential:
credential: str | OpenAIWorkloadIdentityConfig
def _deployment_drop_paths(deployment: Mapping[str, object] | None) -> object:
match deployment:
case {"litellm_params": {"additional_drop_params": object() as additional_drop_params}}:
return additional_drop_params
case _:
return None
def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError:
litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
api_base: Final = litellm_params.get("api_base")
@ -148,6 +156,7 @@ class OpenAITokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat,
output_format: JsonValue,
api_base: str | None,
additional_drop_params: object,
) -> ResponsesCountRequestParts:
match messages_format:
case "anthropic":
@ -157,7 +166,7 @@ class OpenAITokenCounter(BaseTokenCounter):
system=system,
tools=tools,
output_format=output_format,
maps_tool_search=openai_runs_tool_search(model_to_use, api_base),
maps_tool_search=maps_openai_tool_search(model_to_use, api_base, additional_drop_params),
)
case "openai":
input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
@ -237,6 +246,7 @@ class OpenAITokenCounter(BaseTokenCounter):
messages_format=messages_format,
output_format=output_format,
api_base=resolved.api_base,
additional_drop_params=_deployment_drop_paths(deployment),
)
# If no input items were produced (e.g., system-only messages), fall back to local counting

View file

@ -662,6 +662,7 @@ class TestAnthropicMessagesHandlerInputProcessing:
{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False},
{"name": "get_time", "input_schema": {"type": "object", "properties": {}}},
{"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}},
{"name": "get_news", "input_schema": {"type": "object", "properties": {}}, "defer_loading": "true"},
],
}
@ -673,6 +674,7 @@ class TestAnthropicMessagesHandlerInputProcessing:
("get_weather", False, None),
("get_time", None, None),
("get_quote", None, {"defer_loading": True}),
("get_news", None, None),
]
@pytest.mark.asyncio

View file

@ -2,7 +2,7 @@ import datetime
import json
import os
import sys
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Mapping, Sequence
from itertools import chain
from typing import Final, cast
from unittest.mock import AsyncMock, patch
@ -1025,8 +1025,16 @@ async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_
@pytest.mark.asyncio
async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
@pytest.mark.parametrize(
"additional_drop_params",
[
pytest.param(["tools[*].defer_loading"], id="every-tool"),
pytest.param(["tools[1].defer_loading"], id="one-tool-by-index"),
pytest.param(("tools[*].defer_loading",), id="sdk-tuple"),
],
)
async def test_additional_drop_params_of_a_defer_loading_path_answers_400_for_tool_search_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, additional_drop_params: Sequence[str]
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(
@ -1034,7 +1042,7 @@ async def test_additional_drop_params_of_every_defer_loading_answers_400_for_too
monkeypatch,
[TOOL_SEARCH, DEFERRED_WEATHER],
drop_params=False,
additional_drop_params=["tools[*].defer_loading"],
additional_drop_params=additional_drop_params,
)
assert "tool_search_tool_bm25_20251119" in error.value.message

View file

@ -274,6 +274,9 @@ async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool(
[
pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"),
pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"),
pytest.param(
"gpt-5.6-luna", {"additional_drop_params": ["tools[*].defer_loading"]}, id="deployment-drops-deferral"
),
],
)
async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it(