Merge pull request #33886 from BerriAI/litellm_lit4582_cache_control_present

fix(anthropic): only inject cache_control when the request carries none

litellm/integrations/anthropic_cache_control_hook.py

@@ -91,7 +91,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
 
         # Pass through non-message injection points for provider-specific handling
         if remaining_points:
-            non_default_params["cache_control_injection_points"] = remaining_points
+            non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
+                remaining_points
+            )
 
         return model, processed_messages, non_default_params
 
@@ -310,6 +312,35 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             return ChatCompletionCachedContent(type="ephemeral", ttl=ttl)
         return ChatCompletionCachedContent(type="ephemeral")
 
+    @staticmethod
+    def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]:
+        """Mark written-back points as having passed the client cache_control judgment.
+
+        Builds copies because config-owned point dicts are shared across
+        requests; mutating them would leak the stamp into future requests.
+        """
+        return [{**point, "_litellm_judged": True} for point in points]
+
+    @staticmethod
+    def _should_stand_down(
+        points: list[CacheControlInjectionPoint],
+        messages: list[AllMessageValues],
+        system: str | list | None,
+        tools: list | None,
+    ) -> bool:
+        """Whether configured injection points must yield to client-set cache_control.
+
+        Points that a prior pass over this request already judged and wrote
+        back carry the internal judged stamp; any re-entry (acompletion
+        re-entering completion, the async-to-sync /v1/messages dispatch,
+        interceptor sub-calls reusing the request kwargs) must not re-judge
+        them, because by then the messages carry litellm's own injected marks
+        and the judgment would misread those as client breakpoints.
+        """
+        if all(point.get("_litellm_judged") for point in points):
+            return False
+        return AnthropicCacheControlHook._request_has_cache_control(messages, system, tools)
+
     @staticmethod
     def _request_has_cache_control(
         messages: list[AllMessageValues],
@@ -322,7 +353,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         stand down entirely rather than add more, per the auto-caching contract.
         Tools count: they are a breakpoint the client can mark, they count toward
         the provider's four-block limit, and caching only the tool definitions is
-        a common pattern, so injecting alongside them can exceed the cap.
+        a common pattern, so injecting alongside them can exceed the cap. Tools
+        carry the mark either at the top level (Anthropic shape) or nested under
+        ``function`` (OpenAI shape); the Anthropic chat transform accepts both.
         """
         if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages):
             return True
@@ -330,7 +363,14 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system):
                 return True
         if tools is not None:
-            return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools)
+            return any(
+                isinstance(tool, dict)
+                and (
+                    tool.get("cache_control") is not None
+                    or (isinstance(tool.get("function"), dict) and tool["function"].get("cache_control") is not None)
+                )
+                for tool in tools
+            )
         return False
 
     @staticmethod
@@ -392,13 +432,23 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         tools: list | None = None,
     ) -> None:
-        """For /chat/completions: add default injection points to the request params.
+        """For /chat/completions: resolve the injection points the request should carry.
 
-        No-op when injection points are already configured (explicit config wins).
-        Seeding the param lets the existing prompt-management gate and the
-        AnthropicCacheControlHook run unchanged.
+        Configured injection points win over the automatic defaults, but stand
+        down entirely when the client already marked its own cache_control
+        breakpoints (messages or tools): injecting alongside them clashes with
+        the client's caching strategy and can exceed the provider's four-block
+        limit. The judgment happens once per request; points a prior pass
+        wrote back carry the judged stamp and are never re-judged (see
+        ``_should_stand_down``). Seeding the param lets the existing
+        prompt-management gate and the AnthropicCacheControlHook run
+        unchanged.
         """
         if non_default_params.get("cache_control_injection_points"):
+            if AnthropicCacheControlHook._should_stand_down(
+                non_default_params["cache_control_injection_points"], messages, None, tools
+            ):
+                non_default_params.pop("cache_control_injection_points")
             return
         points = AnthropicCacheControlHook.get_default_injection_points(
             messages=messages,
@@ -421,18 +471,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
     ) -> Tuple[List[Dict], str | list | None]:
         """Extract cache_control_injection_points from kwargs and apply if present.
 
-        When none are configured but ``litellm.enable_anthropic_prompt_caching``
-        is on, synthesize default breakpoints for the native /v1/messages path.
-        Pops the key from kwargs; if remaining (non-message) points exist they
-        are written back so downstream transforms can handle them.
+        Configured points stand down entirely when the client already marked
+        its own cache_control breakpoints anywhere in the request. The
+        judgment happens once per request; points a prior pass wrote back
+        carry the judged stamp and are never re-judged (see
+        ``_should_stand_down``). When none are configured but
+        ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default
+        breakpoints for the native /v1/messages path. Pops the key from kwargs;
+        if remaining (non-message) points exist they are written back so
+        downstream transforms can handle them.
         """
+        typed_messages = cast(list[AllMessageValues], messages)  # cast-ok: Anthropic-shaped dicts from v1/messages
         configured = cast(  # cast-ok: kwargs is untyped; this key only holds the documented injection-point list
             list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None)
         )
+        if configured and AnthropicCacheControlHook._should_stand_down(configured, typed_messages, system, tools):
+            return messages, system
         injection_points: list[CacheControlInjectionPoint] = configured or []
         if not injection_points and model is not None:
             injection_points = AnthropicCacheControlHook.get_default_injection_points(
-                messages=cast(list[AllMessageValues], messages),  # cast-ok: Anthropic-shaped dicts from v1/messages
+                messages=typed_messages,
                 system=system,
                 tools=tools,
                 model=model,
@@ -447,7 +505,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             injection_points=injection_points,
         )
         if remaining:
-            kwargs["cache_control_injection_points"] = remaining
+            kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)
         return messages, system
 
     @property

litellm/types/integrations/anthropic_cache_control_hook.py

@@ -1,6 +1,6 @@
 from typing import Literal, Optional, Union
 
-from typing_extensions import TypedDict
+from typing_extensions import NotRequired, TypedDict
 
 from litellm.types.llms.openai import ChatCompletionCachedContent
 
@@ -12,6 +12,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
     role: Optional[Literal["user", "system", "assistant"]]  # Optional: target by role (user, system, assistant)
     index: Optional[Union[int, str]]  # Optional: target by specific index
     control: Optional[ChatCompletionCachedContent]
+    _litellm_judged: NotRequired[bool]  # Internal: written back by litellm once the client cache_control judgment ran
 
 
 class CacheControlToolConfigInjectionPoint(TypedDict):
@@ -19,6 +20,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
 
     location: Literal["tool_config"]
     control: Optional[ChatCompletionCachedContent]
+    _litellm_judged: NotRequired[bool]  # Internal: written back by litellm once the client cache_control judgment ran
 
 
 CacheControlInjectionPoint = Union[

tests/test_litellm/integrations/test_anthropic_cache_control_hook.py

@@ -1265,8 +1265,11 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point():
     )
 
     assert _count_cache_control(processed) == 3
-    # The tool_config point is passed through for the provider transform.
-    assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}]
+    # The tool_config point is passed through for the provider transform,
+    # stamped so re-entries never re-judge it against litellm's own marks.
+    assert non_default_params["cache_control_injection_points"] == [
+        {"location": "tool_config", "_litellm_judged": True}
+    ]
 
 
 @pytest.mark.asyncio
@@ -1622,6 +1625,13 @@ class TestEnableAnthropicPromptCaching:
         monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
         assert [p["index"] for p in self._points(tools=tools)] == [None, -1]
 
+    def test_stands_down_when_tool_function_carries_cache_control(self, monkeypatch):
+        """OpenAI-shaped tools nest cache_control under ``function``; the Anthropic
+        chat transform honors that location, so the stand-down must see it too."""
+        monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
+        tools = [{"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}]
+        assert self._points(tools=tools) == []
+
     def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch):
         """Same guard on the /chat/completions seeding path."""
         monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
@@ -1726,6 +1736,131 @@ class TestEnableAnthropicPromptCaching:
         assert result_msgs == messages
 
 
+class TestConfiguredInjectionPointsStandDown:
+    """Configured cache_control_injection_points must stand down entirely when the
+    client already set its own cache_control anywhere in the request (LIT-4582);
+    injecting alongside client breakpoints clashes with the client's caching
+    strategy and can push the request past Anthropic's four-block limit."""
+
+    CONFIGURED = [{"location": "message", "role": "system"}]
+
+    CLEAN_MESSAGES: List[AllMessageValues] = [
+        {"role": "system", "content": "sys"},
+        {"role": "user", "content": "hi"},
+    ]
+
+    MARKED_MESSAGES: List[AllMessageValues] = [
+        {"role": "system", "content": "sys"},
+        {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]},
+    ]
+
+    V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
+
+    def _seed(self, params, messages, tools=None):
+        AnthropicCacheControlHook.maybe_seed_default_injection_points(
+            non_default_params=params,
+            messages=messages,
+            model="claude-sonnet-4-5",
+            custom_llm_provider="anthropic",
+            tools=tools,
+        )
+
+    def _inject(self, messages, kwargs, system="sys", tools=None):
+        return AnthropicCacheControlHook.maybe_inject_cache_control(
+            messages,
+            system,
+            kwargs,
+            model="claude-sonnet-4-5",
+            custom_llm_provider="anthropic",
+            tools=tools,
+        )
+
+    def test_configured_points_dropped_when_messages_carry_cache_control(self):
+        params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
+        self._seed(params, copy.deepcopy(self.MARKED_MESSAGES))
+        assert "cache_control_injection_points" not in params
+
+    @pytest.mark.parametrize(
+        "tool",
+        [
+            {"type": "function", "function": {"name": "t", "parameters": {}}, "cache_control": {"type": "ephemeral"}},
+            {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}},
+        ],
+        ids=["top_level", "nested_in_function"],
+    )
+    def test_configured_points_dropped_when_tools_carry_cache_control(self, tool):
+        params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
+        self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES), tools=[tool])
+        assert "cache_control_injection_points" not in params
+
+    def test_configured_points_kept_when_request_is_unmarked(self):
+        configured = copy.deepcopy(self.CONFIGURED)
+        params = {"cache_control_injection_points": configured}
+        self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES))
+        assert params["cache_control_injection_points"] is configured
+
+    def test_judged_remainder_survives_reentry_despite_injected_marks(self):
+        """acompletion() re-enters completion() after injection ran, with only the
+        stamped non-message points written back; the re-entry must not misread
+        litellm's own marks as client ones and drop that remainder."""
+        remainder = [{"location": "tool_config", "_litellm_judged": True}]
+        params = {"cache_control_injection_points": remainder}
+        self._seed(params, copy.deepcopy(self.MARKED_MESSAGES))
+        assert params["cache_control_injection_points"] is remainder
+
+    def test_v1_messages_stand_down_when_content_block_marked(self):
+        messages = [
+            {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}
+        ]
+        kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
+        result_msgs, result_sys = self._inject(copy.deepcopy(messages), kwargs)
+        assert result_msgs == messages
+        assert result_sys == "sys"
+        assert "cache_control_injection_points" not in kwargs
+
+    def test_v1_messages_stand_down_when_system_block_marked(self):
+        """A configured point targeting a message must not fire when the client
+        marked the system prompt; the old behavior injected into the message
+        because only the exact targeted position was guarded."""
+        system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]
+        kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]}
+        result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, system=system)
+        assert result_msgs == self.V1_MESSAGES
+        assert result_sys == system
+        assert "cache_control_injection_points" not in kwargs
+
+    def test_v1_messages_stand_down_when_tools_marked(self):
+        tools = [{"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}}]
+        kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
+        result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=tools)
+        assert result_msgs == self.V1_MESSAGES
+        assert result_sys == "sys"
+        assert "cache_control_injection_points" not in kwargs
+
+    def test_v1_messages_configured_points_apply_when_unmarked(self):
+        kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
+        _, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs)
+        assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
+
+    def test_v1_messages_reentry_flow_preserves_tool_config_remainder(self):
+        """The advisor interceptor re-enters anthropic_messages() with the outer
+        request's kwargs and post-injection messages. The first pass applies the
+        message point and writes back a stamped tool_config remainder; the
+        re-entry must keep that remainder even though the messages and system
+        now carry litellm's own marks."""
+        points = [{"location": "message", "role": "system"}, {"location": "tool_config"}]
+        kwargs = {"cache_control_injection_points": copy.deepcopy(points)}
+        msgs1, sys1 = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs)
+        assert sys1[0]["cache_control"] == {"type": "ephemeral"}
+        expected_remainder = [{"location": "tool_config", "_litellm_judged": True}]
+        assert kwargs["cache_control_injection_points"] == expected_remainder
+
+        msgs2, sys2 = self._inject(msgs1, kwargs, system=sys1)
+        assert kwargs["cache_control_injection_points"] == expected_remainder
+        assert msgs2 == msgs1
+        assert sys2 == sys1
+
+
 class TestAnthropicPromptCachingEnvVars:
     """Both settings are read from the environment at import, so an admin can enable
     auto-caching without a config file. Each case re-imports litellm in a subprocess