diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 94c86e07ff5..54f0732fdf1 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -322,7 +322,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): stand down entirely rather than add more, per the auto-caching contract. Tools count: they are a breakpoint the client can mark, they count toward the provider's four-block limit, and caching only the tool definitions is - a common pattern, so injecting alongside them can exceed the cap. + a common pattern, so injecting alongside them can exceed the cap. Tools + carry the mark either at the top level (Anthropic shape) or nested under + ``function`` (OpenAI shape); the Anthropic chat transform accepts both. """ if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): return True @@ -330,7 +332,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): return True if tools is not None: - return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools) + return any( + isinstance(tool, dict) + and ( + tool.get("cache_control") is not None + or (isinstance(tool.get("function"), dict) and tool["function"].get("cache_control") is not None) + ) + for tool in tools + ) return False @staticmethod @@ -391,14 +400,24 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, + is_first_pass: bool = True, ) -> None: - """For /chat/completions: add default injection points to the request params. + """For /chat/completions: resolve the injection points the request should carry. - No-op when injection points are already configured (explicit config wins). - Seeding the param lets the existing prompt-management gate and the - AnthropicCacheControlHook run unchanged. + Configured injection points win over the automatic defaults, but stand + down entirely when the client already marked its own cache_control + breakpoints (messages or tools): injecting alongside them clashes with + the client's caching strategy and can exceed the provider's four-block + limit. Only the first pass over a request may make that judgment; + ``acompletion`` re-enters ``completion`` after injection has already + run, and a later pass would mistake litellm's own injected marks for + client ones and drop the non-message points reserved for provider + transforms. Seeding the param lets the existing prompt-management gate + and the AnthropicCacheControlHook run unchanged. """ if non_default_params.get("cache_control_injection_points"): + if is_first_pass and AnthropicCacheControlHook._request_has_cache_control(messages, None, tools): + non_default_params.pop("cache_control_injection_points") return points = AnthropicCacheControlHook.get_default_injection_points( messages=messages, @@ -418,21 +437,35 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, + is_first_pass: bool = True, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. - When none are configured but ``litellm.enable_anthropic_prompt_caching`` - is on, synthesize default breakpoints for the native /v1/messages path. - Pops the key from kwargs; if remaining (non-message) points exist they - are written back so downstream transforms can handle them. + Configured points stand down entirely when the client already marked + its own cache_control breakpoints anywhere in the request, judged only + on the first pass: the async entry re-dispatches into the sync handler + after injection has run, and a later pass would mistake litellm's own + injected marks for client ones and drop the written-back non-message + points. When none are configured but + ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default + breakpoints for the native /v1/messages path. Pops the key from kwargs; + if remaining (non-message) points exist they are written back so + downstream transforms can handle them. """ + typed_messages = cast(list[AllMessageValues], messages) # cast-ok: Anthropic-shaped dicts from v1/messages configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) + if ( + configured + and is_first_pass + and AnthropicCacheControlHook._request_has_cache_control(typed_messages, system, tools) + ): + return messages, system injection_points: list[CacheControlInjectionPoint] = configured or [] if not injection_points and model is not None: injection_points = AnthropicCacheControlHook.get_default_injection_points( - messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages + messages=typed_messages, system=system, tools=tools, model=model, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 703ccf13c27..92151b45b75 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -351,9 +351,11 @@ async def anthropic_messages( custom_llm_provider=custom_llm_provider, # messages were already empty-text-block sanitized at the top of this # function and are NOT reassigned before this dispatch, so the handler - # can skip its (otherwise redundant) second full-messages scan. Passed - # explicitly (not via **kwargs) so it only affects this direct - # dispatch -- interceptor / sync entry points still sanitize. + # can skip its (otherwise redundant) second full-messages scan. It also + # tells the handler that cache_control injection already judged the + # pristine client input here. Passed explicitly (not via **kwargs) so + # it only affects this direct dispatch -- interceptor / sync entry + # points still sanitize. _litellm_messages_presanitized=True, **kwargs, ) @@ -419,8 +421,12 @@ def anthropic_messages_handler( # protection as the async wrapper. The async wrapper already sanitized and # does not reassign messages before dispatch, so it sets # ``_litellm_messages_presanitized`` to skip this redundant second - # full-messages scan. Pop it so it never leaks into provider params. - if not kwargs.pop("_litellm_messages_presanitized", False): + # full-messages scan. The same flag marks this call as a second pass for + # cache_control injection: the async wrapper already judged the pristine + # client input, and re-judging after injection would misread litellm's own + # marks as client ones. Pop it so it never leaks into provider params. + presanitized = kwargs.pop("_litellm_messages_presanitized", False) + if not presanitized: messages = strip_empty_text_blocks_from_anthropic_messages(messages) messages = sanitize_tool_use_ids_in_anthropic_messages(messages) @@ -429,7 +435,13 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, + system, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + tools=tools, + is_first_pass=not presanitized, ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index 3584297b35f..ab07da7528b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5080,6 +5080,7 @@ def completion( # type: ignore model=model, custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, + is_first_pass=not acompletion, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 70c1f65b541..85e77f2f493 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1622,6 +1622,13 @@ class TestEnableAnthropicPromptCaching: monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) assert [p["index"] for p in self._points(tools=tools)] == [None, -1] + def test_stands_down_when_tool_function_carries_cache_control(self, monkeypatch): + """OpenAI-shaped tools nest cache_control under ``function``; the Anthropic + chat transform honors that location, so the stand-down must see it too.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + tools = [{"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}] + assert self._points(tools=tools) == [] + def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): """Same guard on the /chat/completions seeding path.""" monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) @@ -1726,6 +1733,128 @@ class TestEnableAnthropicPromptCaching: assert result_msgs == messages +class TestConfiguredInjectionPointsStandDown: + """Configured cache_control_injection_points must stand down entirely when the + client already set its own cache_control anywhere in the request (LIT-4582); + injecting alongside client breakpoints clashes with the client's caching + strategy and can push the request past Anthropic's four-block limit.""" + + CONFIGURED = [{"location": "message", "role": "system"}] + + CLEAN_MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "hi"}, + ] + + MARKED_MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}, + ] + + V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + def _seed(self, params, messages, tools=None, is_first_pass=True): + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=messages, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=tools, + is_first_pass=is_first_pass, + ) + + def _inject(self, messages, kwargs, system="sys", tools=None, is_first_pass=True): + return AnthropicCacheControlHook.maybe_inject_cache_control( + messages, + system, + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=tools, + is_first_pass=is_first_pass, + ) + + def test_configured_points_dropped_when_messages_carry_cache_control(self): + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + self._seed(params, copy.deepcopy(self.MARKED_MESSAGES)) + assert "cache_control_injection_points" not in params + + @pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "t", "parameters": {}}, "cache_control": {"type": "ephemeral"}}, + {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}, + ], + ids=["top_level", "nested_in_function"], + ) + def test_configured_points_dropped_when_tools_carry_cache_control(self, tool): + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES), tools=[tool]) + assert "cache_control_injection_points" not in params + + def test_configured_points_kept_when_request_is_unmarked(self): + configured = copy.deepcopy(self.CONFIGURED) + params = {"cache_control_injection_points": configured} + self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES)) + assert params["cache_control_injection_points"] is configured + + def test_second_pass_keeps_points_despite_injected_marks(self): + """acompletion() re-enters completion() after injection ran, with only the + non-message points written back; the second pass must not misread litellm's + own marks as client ones and drop that remainder.""" + remainder = [{"location": "tool_config"}] + params = {"cache_control_injection_points": remainder} + self._seed(params, copy.deepcopy(self.MARKED_MESSAGES), is_first_pass=False) + assert params["cache_control_injection_points"] is remainder + + def test_v1_messages_stand_down_when_content_block_marked(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} + ] + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + result_msgs, result_sys = self._inject(copy.deepcopy(messages), kwargs) + assert result_msgs == messages + assert result_sys == "sys" + assert "cache_control_injection_points" not in kwargs + + def test_v1_messages_stand_down_when_system_block_marked(self): + """A configured point targeting a message must not fire when the client + marked the system prompt; the old behavior injected into the message + because only the exact targeted position was guarded.""" + system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, system=system) + assert result_msgs == self.V1_MESSAGES + assert result_sys == system + assert "cache_control_injection_points" not in kwargs + + def test_v1_messages_stand_down_when_tools_marked(self): + tools = [{"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}}] + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=tools) + assert result_msgs == self.V1_MESSAGES + assert result_sys == "sys" + assert "cache_control_injection_points" not in kwargs + + def test_v1_messages_configured_points_apply_when_unmarked(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + _, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) + assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + + def test_v1_messages_second_pass_writes_back_remainder(self): + """The async entry re-dispatches into the sync handler after injecting; the + surviving tool_config remainder must survive that second pass even though + the messages now carry litellm's own marks.""" + marked = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} + ] + remainder = [{"location": "tool_config"}] + kwargs = {"cache_control_injection_points": remainder} + result_msgs, _ = self._inject(copy.deepcopy(marked), kwargs, is_first_pass=False) + assert result_msgs == marked + assert kwargs["cache_control_injection_points"] == remainder + + class TestAnthropicPromptCachingEnvVars: """Both settings are read from the environment at import, so an admin can enable auto-caching without a config file. Each case re-imports litellm in a subprocess