mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(anthropic): keep tool search off for any defer_loading drop path and restore only boolean deferrals
A drop path such as tools[1].defer_loading or the SDK's tuple form now turns the tool search mapping off like tools[*].defer_loading, the count path reads the deployment's drop list, and a guardrail write-back restores only a boolean defer_loading
This commit is contained in:
parent
06dcd798cf
commit
64f1ab196b
7 changed files with 57 additions and 20 deletions
|
|
@ -259,10 +259,10 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s
|
|||
|
||||
|
||||
def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
if "defer_loading" in tool:
|
||||
return {"defer_loading": tool["defer_loading"]}
|
||||
match tool.get("custom"):
|
||||
case {"defer_loading": bool() as wrapped}:
|
||||
match tool:
|
||||
case {"defer_loading": bool() as deferred}:
|
||||
return {"defer_loading": deferred}
|
||||
case {"custom": {"defer_loading": bool() as wrapped}}:
|
||||
return {"custom": {"defer_loading": wrapped}}
|
||||
case _:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text
|
|||
from ..safeguards import with_safeguard_results
|
||||
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
|
||||
from .streaming_iterator import AnthropicResponsesStreamWrapper
|
||||
from .tool_translation import openai_runs_tool_search
|
||||
from .tool_translation import maps_openai_tool_search
|
||||
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
|
||||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
|
|
@ -183,13 +183,12 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
|
|||
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
|
||||
|
||||
|
||||
_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading"
|
||||
|
||||
|
||||
def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
|
||||
provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider"))
|
||||
return provider == LlmProviders.OPENAI.value and openai_runs_tool_search(
|
||||
local_model_name(model, provider), forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url")
|
||||
return provider == LlmProviders.OPENAI.value and maps_openai_tool_search(
|
||||
local_model_name(model, provider),
|
||||
forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
|
||||
forwarded_kwargs.get("additional_drop_params"),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -337,7 +336,7 @@ def _build_responses_kwargs(
|
|||
),
|
||||
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
|
||||
drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")),
|
||||
maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped,
|
||||
maps_tool_search=_maps_tool_search(model, forwarded_kwargs),
|
||||
)
|
||||
|
||||
# Normalize reasoning effort based on model capabilities
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, replace
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
|
|
@ -13,6 +15,9 @@ from litellm.types.utils import LlmProviders
|
|||
|
||||
_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P<family>[a-z0-9_]+?)_(?P<version>\d{8})")
|
||||
_UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"})
|
||||
_DEFERRAL_DROP_PATH: Final = re.compile(r"tools\[[^\]]*\]\.defer_loading")
|
||||
_DropPaths: TypeAlias = list[object] | tuple[object, ...]
|
||||
_DROP_PATHS: Final[TypeAdapter[_DropPaths]] = TypeAdapter(_DropPaths, config=ConfigDict(strict=True))
|
||||
_TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429"
|
||||
_COMPUTER_MODIFIERS_SINCE: Final = "20250124"
|
||||
_COMPUTER_ZOOM_SINCE: Final = "20251124"
|
||||
|
|
@ -342,7 +347,17 @@ def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError:
|
|||
)
|
||||
|
||||
|
||||
def openai_runs_tool_search(model: str, api_base: object) -> bool:
|
||||
return targets_openai_api(api_base) and litellm.supports_tool_search(
|
||||
model=model, custom_llm_provider=LlmProviders.OPENAI.value
|
||||
def _drops_deferral(additional_drop_params: object) -> bool:
|
||||
try:
|
||||
paths: Final = _DROP_PATHS.validate_python(additional_drop_params)
|
||||
except ValidationError:
|
||||
return False
|
||||
return any(isinstance(path, str) and _DEFERRAL_DROP_PATH.fullmatch(path) is not None for path in paths)
|
||||
|
||||
|
||||
def maps_openai_tool_search(model: str, api_base: object, additional_drop_params: object) -> bool:
|
||||
return (
|
||||
targets_openai_api(api_base)
|
||||
and not _drops_deferral(additional_drop_params)
|
||||
and litellm.supports_tool_search(model=model, custom_llm_provider=LlmProviders.OPENAI.value)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy
|
|||
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
|
||||
from litellm.llms.anthropic.messages.responses_adapters.tool_translation import openai_runs_tool_search
|
||||
from litellm.llms.anthropic.messages.responses_adapters.tool_translation import maps_openai_tool_search
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
|
|
@ -64,6 +64,14 @@ class _CountCredential:
|
|||
credential: str | OpenAIWorkloadIdentityConfig
|
||||
|
||||
|
||||
def _deployment_drop_paths(deployment: Mapping[str, object] | None) -> object:
|
||||
match deployment:
|
||||
case {"litellm_params": {"additional_drop_params": object() as additional_drop_params}}:
|
||||
return additional_drop_params
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError:
|
||||
litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
|
||||
api_base: Final = litellm_params.get("api_base")
|
||||
|
|
@ -148,6 +156,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat,
|
||||
output_format: JsonValue,
|
||||
api_base: str | None,
|
||||
additional_drop_params: object,
|
||||
) -> ResponsesCountRequestParts:
|
||||
match messages_format:
|
||||
case "anthropic":
|
||||
|
|
@ -157,7 +166,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
system=system,
|
||||
tools=tools,
|
||||
output_format=output_format,
|
||||
maps_tool_search=openai_runs_tool_search(model_to_use, api_base),
|
||||
maps_tool_search=maps_openai_tool_search(model_to_use, api_base, additional_drop_params),
|
||||
)
|
||||
case "openai":
|
||||
input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
|
||||
|
|
@ -237,6 +246,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
api_base=resolved.api_base,
|
||||
additional_drop_params=_deployment_drop_paths(deployment),
|
||||
)
|
||||
|
||||
# If no input items were produced (e.g., system-only messages), fall back to local counting
|
||||
|
|
|
|||
|
|
@ -662,6 +662,7 @@ class TestAnthropicMessagesHandlerInputProcessing:
|
|||
{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False},
|
||||
{"name": "get_time", "input_schema": {"type": "object", "properties": {}}},
|
||||
{"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}},
|
||||
{"name": "get_news", "input_schema": {"type": "object", "properties": {}}, "defer_loading": "true"},
|
||||
],
|
||||
}
|
||||
|
||||
|
|
@ -673,6 +674,7 @@ class TestAnthropicMessagesHandlerInputProcessing:
|
|||
("get_weather", False, None),
|
||||
("get_time", None, None),
|
||||
("get_quote", None, {"defer_loading": True}),
|
||||
("get_news", None, None),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import datetime
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
|
@ -1025,8 +1025,16 @@ async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
@pytest.mark.parametrize(
|
||||
"additional_drop_params",
|
||||
[
|
||||
pytest.param(["tools[*].defer_loading"], id="every-tool"),
|
||||
pytest.param(["tools[1].defer_loading"], id="one-tool-by-index"),
|
||||
pytest.param(("tools[*].defer_loading",), id="sdk-tuple"),
|
||||
],
|
||||
)
|
||||
async def test_additional_drop_params_of_a_defer_loading_path_answers_400_for_tool_search_without_drop_params(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, additional_drop_params: Sequence[str]
|
||||
):
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as error:
|
||||
await _openai_request_for(
|
||||
|
|
@ -1034,7 +1042,7 @@ async def test_additional_drop_params_of_every_defer_loading_answers_400_for_too
|
|||
monkeypatch,
|
||||
[TOOL_SEARCH, DEFERRED_WEATHER],
|
||||
drop_params=False,
|
||||
additional_drop_params=["tools[*].defer_loading"],
|
||||
additional_drop_params=additional_drop_params,
|
||||
)
|
||||
|
||||
assert "tool_search_tool_bm25_20251119" in error.value.message
|
||||
|
|
|
|||
|
|
@ -274,6 +274,9 @@ async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool(
|
|||
[
|
||||
pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"),
|
||||
pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"),
|
||||
pytest.param(
|
||||
"gpt-5.6-luna", {"additional_drop_params": ["tools[*].defer_loading"]}, id="deployment-drops-deferral"
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue