diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 026d8b8e82e..79ed48943b3 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -311,16 +311,26 @@ class AnthropicCacheControlHook(CustomPromptManagement): return ChatCompletionCachedContent(type="ephemeral") @staticmethod - def _request_has_cache_control(messages: list[AllMessageValues], system: Optional[Union[str, list]]) -> bool: + def _request_has_cache_control( + messages: list[AllMessageValues], + system: Optional[Union[str, list]], + tools: Optional[list] = None, + ) -> bool: """Return True if the request already carries any client-supplied cache_control. When the client (e.g. Claude Code) already marks its own breakpoints we stand down entirely rather than add more, per the auto-caching contract. + Tools count: they are a breakpoint the client can mark, they count toward + the provider's four-block limit, and caching only the tool definitions is + a common pattern, so injecting alongside them can exceed the cap. """ if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): return True if isinstance(system, list): - return any(isinstance(block, dict) and block.get("cache_control") is not None for block in system) + if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): + return True + if tools is not None: + return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools) return False @staticmethod @@ -329,6 +339,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): system: Optional[Union[str, list]], model: str, custom_llm_provider: Optional[str], + tools: Optional[list] = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. @@ -363,7 +374,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not supports_prompt_caching(model=model, custom_llm_provider=provider): return [] - if AnthropicCacheControlHook._request_has_cache_control(messages, system): + if AnthropicCacheControlHook._request_has_cache_control(messages, system, tools): return [] control = AnthropicCacheControlHook._default_control() @@ -379,6 +390,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): messages: list[AllMessageValues], model: str, custom_llm_provider: Optional[str], + tools: Optional[list] = None, ) -> None: """For /chat/completions: add default injection points to the request params. @@ -393,6 +405,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): system=None, model=model, custom_llm_provider=custom_llm_provider, + tools=tools, ) if points: non_default_params["cache_control_injection_points"] = points @@ -404,6 +417,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): kwargs: Dict[str, Any], model: Optional[str] = None, custom_llm_provider: Optional[str] = None, + tools: Optional[list[dict]] = None, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. @@ -420,6 +434,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): injection_points = AnthropicCacheControlHook.get_default_injection_points( messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages system=system, + tools=tools, model=model, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index c205d7516e6..59eedbe4538 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -237,7 +237,7 @@ async def anthropic_messages( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools ) original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -428,7 +428,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index 4a9b5bdc76f..3584297b35f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -521,6 +521,7 @@ async def acompletion( messages=cast(list[AllMessageValues], messages), # cast-ok: acompletion types messages as a bare List model=model, custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs + tools=tools, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5078,6 +5079,7 @@ def completion( # type: ignore messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List model=model, custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs + tools=tools, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 6a67f1d6643..70c1f65b541 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1547,12 +1547,13 @@ class TestEnableAnthropicPromptCaching: {"role": "user", "content": "latest turn"}, ] - def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None): + def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None, tools=None): return AnthropicCacheControlHook.get_default_injection_points( messages=copy.deepcopy(self.MESSAGES) if messages is None else messages, system=system, model=model, custom_llm_provider=provider, + tools=tools, ) def test_disabled_by_default(self): @@ -1597,6 +1598,58 @@ class TestEnableAnthropicPromptCaching: system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] assert self._points(messages=[{"role": "user", "content": "hi"}], system=system) == [] + @staticmethod + def _tools(count: int, cached: bool) -> List[dict]: + tool: dict = {"type": "function", "function": {"name": "t", "description": "d", "parameters": {}}} + if cached: + tool["cache_control"] = {"type": "ephemeral"} + return [{**tool, "function": {**tool["function"], "name": f"t{i}"}} for i in range(count)] + + def test_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): + """Caching just the tool definitions is a normal client pattern, and those + breakpoints count toward the provider's four-block limit. Three of them plus + our two would be five, which Anthropic rejects outright.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert self._points(tools=self._tools(3, cached=True)) == [] + + def test_injects_when_tools_carry_no_cache_control(self, monkeypatch): + """Tools alone must not suppress injection; only client-marked ones do.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert [p["index"] for p in self._points(tools=self._tools(3, cached=False))] == [None, -1] + + @pytest.mark.parametrize("tools", [None, []]) + def test_absent_tools_do_not_suppress_injection(self, monkeypatch, tools): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert [p["index"] for p in self._points(tools=tools)] == [None, -1] + + def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): + """Same guard on the /chat/completions seeding path.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=self._tools(3, cached=True), + ) + assert "cache_control_injection_points" not in params + + def test_v1_messages_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): + """Same guard on the /v1/messages path, where tools reach the hook directly.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), + "sys", + {}, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=self._tools(3, cached=True), + ) + assert result_sys == "sys" + assert result_msgs == messages + def test_default_ttl_is_anthropics_five_minute_cache(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) assert all(p["control"] == {"type": "ephemeral"} for p in self._points())