diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b06372baa78..6f90acade10 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -33,6 +33,7 @@ from litellm.types.integrations.anthropic_cache_control_hook import ( CacheControlMessageInjectionPoint, ) from litellm.types.llms.anthropic import ( + ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, AnthropicSystemMessageContent, ) @@ -127,6 +128,10 @@ def _tool_carries_cache_breakpoint(tool: object) -> bool: ) +def _chat_transform_drops_tool_cache_control(tool: object) -> bool: + return isinstance(tool, dict) and tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + + def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES @@ -303,9 +308,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): """Client breakpoints outside messages and system that the provider cap still counts. A tool carries its mark at the top level (Anthropic shape) or under ``function`` - (OpenAI shape); the Anthropic chat transform forwards both. A top-level - ``cache_control`` is Anthropic's automatic caching, which places one breakpoint - of its own on top of the explicit ones. + (OpenAI shape). A top-level ``cache_control`` is Anthropic's automatic caching, + which places one breakpoint of its own on top of the explicit ones. Callers + pass only the tools whose mark reaches the provider on their path. """ automatic_blocks: Final = 1 if cache_control is not None else 0 tool_blocks: Final = sum(1 for tool in tools if _tool_carries_cache_breakpoint(tool)) if tools else 0 @@ -786,10 +791,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): """ configured: Final = non_default_params.get("cache_control_injection_points") if configured: + tools_keeping_marks: Final = tuple( + tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool) + ) non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_for_prompt_hook( configured, AnthropicCacheControlHook.count_external_cache_breakpoints( - tools, non_default_params.get("cache_control") + tools_keeping_marks, non_default_params.get("cache_control") ), model, custom_llm_provider, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index bcd24695f25..43a7b0e0e9c 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -753,6 +753,10 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) ANTHROPIC_TOOL_SEARCH_BETA_HEADER: Final = "advanced-tool-use-2025-11-20" +ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( + {"tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"} +) + # Effort beta header constant ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 3424cc5fed6..1dfd9cf619b 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2114,6 +2114,16 @@ class TestConfiguredInjectionPointsSurviveClientMarks: UNMARKED_TOOL = {"type": "function", "function": {"name": "t", "parameters": {}}} MARKED_V1_TOOL = {"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}} UNMARKED_V1_TOOL = {"name": "t", "input_schema": {}} + MARKED_TOOL_SEARCH_REGEX = { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search", + "cache_control": {"type": "ephemeral"}, + } + MARKED_TOOL_SEARCH_BM25 = { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search", + "cache_control": {"type": "ephemeral"}, + } @staticmethod def _marked_user_turns(count): @@ -2199,6 +2209,17 @@ class TestConfiguredInjectionPointsSurviveClientMarks: processed = self._chat(params, copy.deepcopy(messages)) assert _count_cache_control(processed) == 3 + injected + @pytest.mark.parametrize("tool", [MARKED_TOOL_SEARCH_REGEX, MARKED_TOOL_SEARCH_BM25], ids=["regex", "bm25"]) + def test_chat_cap_ignores_marked_tool_search_tools(self, tool): + """The chat transform strips cache_control from tool-search tools before the + request leaves, so a client mark there never reaches the provider's cap and + must not cost the configured point its fourth slot.""" + messages = [{"role": "system", "content": "sys"}, *self._marked_user_turns(3)] + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + self._seed(params, copy.deepcopy(messages), tools=[tool]) + processed = self._chat(params, copy.deepcopy(messages)) + assert _count_cache_control(processed) == 4 + @pytest.mark.parametrize("marked_turns,injected", [(2, 1), (3, 0)]) def test_chat_root_cache_control_reserves_a_slot(self, marked_turns, injected): """Anthropic's automatic caching (a top-level ``cache_control``) places one @@ -2261,9 +2282,11 @@ class TestConfiguredInjectionPointsSurviveClientMarks: "tool,expected_system", [ (MARKED_V1_TOOL, "sys"), + (MARKED_TOOL_SEARCH_REGEX, "sys"), + (MARKED_TOOL_SEARCH_BM25, "sys"), (UNMARKED_V1_TOOL, [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), ], - ids=["marked", "unmarked"], + ids=["marked", "marked_tool_search_regex", "marked_tool_search_bm25", "unmarked"], ) def test_v1_messages_cap_counts_client_marked_tools(self, tool, expected_system): kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}