From 0268d0151636b7367d23ad5dcfe0e5448d7553f7 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 18 Jul 2026 16:44:27 -0700 Subject: [PATCH 1/3] fix(anthropic): only inject cache_control when the request carries none --- .../anthropic_cache_control_hook.py | 55 ++++++-- .../messages/handler.py | 24 +++- litellm/main.py | 1 + .../test_anthropic_cache_control_hook.py | 129 ++++++++++++++++++ 4 files changed, 192 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 94c86e07ff5..54f0732fdf1 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -322,7 +322,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): stand down entirely rather than add more, per the auto-caching contract. Tools count: they are a breakpoint the client can mark, they count toward the provider's four-block limit, and caching only the tool definitions is - a common pattern, so injecting alongside them can exceed the cap. + a common pattern, so injecting alongside them can exceed the cap. Tools + carry the mark either at the top level (Anthropic shape) or nested under + ``function`` (OpenAI shape); the Anthropic chat transform accepts both. """ if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): return True @@ -330,7 +332,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): return True if tools is not None: - return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools) + return any( + isinstance(tool, dict) + and ( + tool.get("cache_control") is not None + or (isinstance(tool.get("function"), dict) and tool["function"].get("cache_control") is not None) + ) + for tool in tools + ) return False @staticmethod @@ -391,14 +400,24 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, + is_first_pass: bool = True, ) -> None: - """For /chat/completions: add default injection points to the request params. + """For /chat/completions: resolve the injection points the request should carry. - No-op when injection points are already configured (explicit config wins). - Seeding the param lets the existing prompt-management gate and the - AnthropicCacheControlHook run unchanged. + Configured injection points win over the automatic defaults, but stand + down entirely when the client already marked its own cache_control + breakpoints (messages or tools): injecting alongside them clashes with + the client's caching strategy and can exceed the provider's four-block + limit. Only the first pass over a request may make that judgment; + ``acompletion`` re-enters ``completion`` after injection has already + run, and a later pass would mistake litellm's own injected marks for + client ones and drop the non-message points reserved for provider + transforms. Seeding the param lets the existing prompt-management gate + and the AnthropicCacheControlHook run unchanged. """ if non_default_params.get("cache_control_injection_points"): + if is_first_pass and AnthropicCacheControlHook._request_has_cache_control(messages, None, tools): + non_default_params.pop("cache_control_injection_points") return points = AnthropicCacheControlHook.get_default_injection_points( messages=messages, @@ -418,21 +437,35 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, + is_first_pass: bool = True, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. - When none are configured but ``litellm.enable_anthropic_prompt_caching`` - is on, synthesize default breakpoints for the native /v1/messages path. - Pops the key from kwargs; if remaining (non-message) points exist they - are written back so downstream transforms can handle them. + Configured points stand down entirely when the client already marked + its own cache_control breakpoints anywhere in the request, judged only + on the first pass: the async entry re-dispatches into the sync handler + after injection has run, and a later pass would mistake litellm's own + injected marks for client ones and drop the written-back non-message + points. When none are configured but + ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default + breakpoints for the native /v1/messages path. Pops the key from kwargs; + if remaining (non-message) points exist they are written back so + downstream transforms can handle them. """ + typed_messages = cast(list[AllMessageValues], messages) # cast-ok: Anthropic-shaped dicts from v1/messages configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) + if ( + configured + and is_first_pass + and AnthropicCacheControlHook._request_has_cache_control(typed_messages, system, tools) + ): + return messages, system injection_points: list[CacheControlInjectionPoint] = configured or [] if not injection_points and model is not None: injection_points = AnthropicCacheControlHook.get_default_injection_points( - messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages + messages=typed_messages, system=system, tools=tools, model=model, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 703ccf13c27..92151b45b75 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -351,9 +351,11 @@ async def anthropic_messages( custom_llm_provider=custom_llm_provider, # messages were already empty-text-block sanitized at the top of this # function and are NOT reassigned before this dispatch, so the handler - # can skip its (otherwise redundant) second full-messages scan. Passed - # explicitly (not via **kwargs) so it only affects this direct - # dispatch -- interceptor / sync entry points still sanitize. + # can skip its (otherwise redundant) second full-messages scan. It also + # tells the handler that cache_control injection already judged the + # pristine client input here. Passed explicitly (not via **kwargs) so + # it only affects this direct dispatch -- interceptor / sync entry + # points still sanitize. _litellm_messages_presanitized=True, **kwargs, ) @@ -419,8 +421,12 @@ def anthropic_messages_handler( # protection as the async wrapper. The async wrapper already sanitized and # does not reassign messages before dispatch, so it sets # ``_litellm_messages_presanitized`` to skip this redundant second - # full-messages scan. Pop it so it never leaks into provider params. - if not kwargs.pop("_litellm_messages_presanitized", False): + # full-messages scan. The same flag marks this call as a second pass for + # cache_control injection: the async wrapper already judged the pristine + # client input, and re-judging after injection would misread litellm's own + # marks as client ones. Pop it so it never leaks into provider params. + presanitized = kwargs.pop("_litellm_messages_presanitized", False) + if not presanitized: messages = strip_empty_text_blocks_from_anthropic_messages(messages) messages = sanitize_tool_use_ids_in_anthropic_messages(messages) @@ -429,7 +435,13 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, + system, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + tools=tools, + is_first_pass=not presanitized, ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index 3584297b35f..ab07da7528b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5080,6 +5080,7 @@ def completion( # type: ignore model=model, custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, + is_first_pass=not acompletion, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 70c1f65b541..85e77f2f493 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1622,6 +1622,13 @@ class TestEnableAnthropicPromptCaching: monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) assert [p["index"] for p in self._points(tools=tools)] == [None, -1] + def test_stands_down_when_tool_function_carries_cache_control(self, monkeypatch): + """OpenAI-shaped tools nest cache_control under ``function``; the Anthropic + chat transform honors that location, so the stand-down must see it too.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + tools = [{"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}] + assert self._points(tools=tools) == [] + def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): """Same guard on the /chat/completions seeding path.""" monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) @@ -1726,6 +1733,128 @@ class TestEnableAnthropicPromptCaching: assert result_msgs == messages +class TestConfiguredInjectionPointsStandDown: + """Configured cache_control_injection_points must stand down entirely when the + client already set its own cache_control anywhere in the request (LIT-4582); + injecting alongside client breakpoints clashes with the client's caching + strategy and can push the request past Anthropic's four-block limit.""" + + CONFIGURED = [{"location": "message", "role": "system"}] + + CLEAN_MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "hi"}, + ] + + MARKED_MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}, + ] + + V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + def _seed(self, params, messages, tools=None, is_first_pass=True): + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=messages, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=tools, + is_first_pass=is_first_pass, + ) + + def _inject(self, messages, kwargs, system="sys", tools=None, is_first_pass=True): + return AnthropicCacheControlHook.maybe_inject_cache_control( + messages, + system, + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + tools=tools, + is_first_pass=is_first_pass, + ) + + def test_configured_points_dropped_when_messages_carry_cache_control(self): + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + self._seed(params, copy.deepcopy(self.MARKED_MESSAGES)) + assert "cache_control_injection_points" not in params + + @pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "t", "parameters": {}}, "cache_control": {"type": "ephemeral"}}, + {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}, + ], + ids=["top_level", "nested_in_function"], + ) + def test_configured_points_dropped_when_tools_carry_cache_control(self, tool): + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES), tools=[tool]) + assert "cache_control_injection_points" not in params + + def test_configured_points_kept_when_request_is_unmarked(self): + configured = copy.deepcopy(self.CONFIGURED) + params = {"cache_control_injection_points": configured} + self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES)) + assert params["cache_control_injection_points"] is configured + + def test_second_pass_keeps_points_despite_injected_marks(self): + """acompletion() re-enters completion() after injection ran, with only the + non-message points written back; the second pass must not misread litellm's + own marks as client ones and drop that remainder.""" + remainder = [{"location": "tool_config"}] + params = {"cache_control_injection_points": remainder} + self._seed(params, copy.deepcopy(self.MARKED_MESSAGES), is_first_pass=False) + assert params["cache_control_injection_points"] is remainder + + def test_v1_messages_stand_down_when_content_block_marked(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} + ] + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + result_msgs, result_sys = self._inject(copy.deepcopy(messages), kwargs) + assert result_msgs == messages + assert result_sys == "sys" + assert "cache_control_injection_points" not in kwargs + + def test_v1_messages_stand_down_when_system_block_marked(self): + """A configured point targeting a message must not fire when the client + marked the system prompt; the old behavior injected into the message + because only the exact targeted position was guarded.""" + system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, system=system) + assert result_msgs == self.V1_MESSAGES + assert result_sys == system + assert "cache_control_injection_points" not in kwargs + + def test_v1_messages_stand_down_when_tools_marked(self): + tools = [{"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}}] + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=tools) + assert result_msgs == self.V1_MESSAGES + assert result_sys == "sys" + assert "cache_control_injection_points" not in kwargs + + def test_v1_messages_configured_points_apply_when_unmarked(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + _, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) + assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + + def test_v1_messages_second_pass_writes_back_remainder(self): + """The async entry re-dispatches into the sync handler after injecting; the + surviving tool_config remainder must survive that second pass even though + the messages now carry litellm's own marks.""" + marked = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} + ] + remainder = [{"location": "tool_config"}] + kwargs = {"cache_control_injection_points": remainder} + result_msgs, _ = self._inject(copy.deepcopy(marked), kwargs, is_first_pass=False) + assert result_msgs == marked + assert kwargs["cache_control_injection_points"] == remainder + + class TestAnthropicPromptCachingEnvVars: """Both settings are read from the environment at import, so an admin can enable auto-caching without a config file. Each case re-imports litellm in a subprocess From ecff0c0a7de44c890b2c59a0bf6815b2789ddbc5 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 18 Jul 2026 16:53:44 -0700 Subject: [PATCH 2/3] fix(anthropic): state the async entry's first-pass judgment explicitly --- litellm/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/main.py b/litellm/main.py index ab07da7528b..f96064c0591 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -522,6 +522,7 @@ async def acompletion( model=model, custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, + is_first_pass=True, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( From e0648571ed9dc6d31337e975f0a55c36a77d985d Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 18 Jul 2026 17:11:03 -0700 Subject: [PATCH 3/3] fix(anthropic): carry the stand-down judgment inside written-back injection points --- .../anthropic_cache_control_hook.py | 67 +++++++++++++------ .../messages/handler.py | 24 ++----- litellm/main.py | 2 - .../anthropic_cache_control_hook.py | 4 +- .../test_anthropic_cache_control_hook.py | 52 +++++++------- 5 files changed, 84 insertions(+), 65 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 54f0732fdf1..faedf8ae1a3 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -91,7 +91,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Pass through non-message injection points for provider-specific handling if remaining_points: - non_default_params["cache_control_injection_points"] = remaining_points + non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged( + remaining_points + ) return model, processed_messages, non_default_params @@ -310,6 +312,35 @@ class AnthropicCacheControlHook(CustomPromptManagement): return ChatCompletionCachedContent(type="ephemeral", ttl=ttl) return ChatCompletionCachedContent(type="ephemeral") + @staticmethod + def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]: + """Mark written-back points as having passed the client cache_control judgment. + + Builds copies because config-owned point dicts are shared across + requests; mutating them would leak the stamp into future requests. + """ + return [{**point, "_litellm_judged": True} for point in points] + + @staticmethod + def _should_stand_down( + points: list[CacheControlInjectionPoint], + messages: list[AllMessageValues], + system: str | list | None, + tools: list | None, + ) -> bool: + """Whether configured injection points must yield to client-set cache_control. + + Points that a prior pass over this request already judged and wrote + back carry the internal judged stamp; any re-entry (acompletion + re-entering completion, the async-to-sync /v1/messages dispatch, + interceptor sub-calls reusing the request kwargs) must not re-judge + them, because by then the messages carry litellm's own injected marks + and the judgment would misread those as client breakpoints. + """ + if all(point.get("_litellm_judged") for point in points): + return False + return AnthropicCacheControlHook._request_has_cache_control(messages, system, tools) + @staticmethod def _request_has_cache_control( messages: list[AllMessageValues], @@ -400,7 +431,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, - is_first_pass: bool = True, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -408,15 +438,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): down entirely when the client already marked its own cache_control breakpoints (messages or tools): injecting alongside them clashes with the client's caching strategy and can exceed the provider's four-block - limit. Only the first pass over a request may make that judgment; - ``acompletion`` re-enters ``completion`` after injection has already - run, and a later pass would mistake litellm's own injected marks for - client ones and drop the non-message points reserved for provider - transforms. Seeding the param lets the existing prompt-management gate - and the AnthropicCacheControlHook run unchanged. + limit. The judgment happens once per request; points a prior pass + wrote back carry the judged stamp and are never re-judged (see + ``_should_stand_down``). Seeding the param lets the existing + prompt-management gate and the AnthropicCacheControlHook run + unchanged. """ if non_default_params.get("cache_control_injection_points"): - if is_first_pass and AnthropicCacheControlHook._request_has_cache_control(messages, None, tools): + if AnthropicCacheControlHook._should_stand_down( + non_default_params["cache_control_injection_points"], messages, None, tools + ): non_default_params.pop("cache_control_injection_points") return points = AnthropicCacheControlHook.get_default_injection_points( @@ -437,16 +468,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, - is_first_pass: bool = True, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. Configured points stand down entirely when the client already marked - its own cache_control breakpoints anywhere in the request, judged only - on the first pass: the async entry re-dispatches into the sync handler - after injection has run, and a later pass would mistake litellm's own - injected marks for client ones and drop the written-back non-message - points. When none are configured but + its own cache_control breakpoints anywhere in the request. The + judgment happens once per request; points a prior pass wrote back + carry the judged stamp and are never re-judged (see + ``_should_stand_down``). When none are configured but ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default breakpoints for the native /v1/messages path. Pops the key from kwargs; if remaining (non-message) points exist they are written back so @@ -456,11 +485,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) - if ( - configured - and is_first_pass - and AnthropicCacheControlHook._request_has_cache_control(typed_messages, system, tools) - ): + if configured and AnthropicCacheControlHook._should_stand_down(configured, typed_messages, system, tools): return messages, system injection_points: list[CacheControlInjectionPoint] = configured or [] if not injection_points and model is not None: @@ -480,7 +505,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): injection_points=injection_points, ) if remaining: - kwargs["cache_control_injection_points"] = remaining + kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining) return messages, system @property diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 92151b45b75..703ccf13c27 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -351,11 +351,9 @@ async def anthropic_messages( custom_llm_provider=custom_llm_provider, # messages were already empty-text-block sanitized at the top of this # function and are NOT reassigned before this dispatch, so the handler - # can skip its (otherwise redundant) second full-messages scan. It also - # tells the handler that cache_control injection already judged the - # pristine client input here. Passed explicitly (not via **kwargs) so - # it only affects this direct dispatch -- interceptor / sync entry - # points still sanitize. + # can skip its (otherwise redundant) second full-messages scan. Passed + # explicitly (not via **kwargs) so it only affects this direct + # dispatch -- interceptor / sync entry points still sanitize. _litellm_messages_presanitized=True, **kwargs, ) @@ -421,12 +419,8 @@ def anthropic_messages_handler( # protection as the async wrapper. The async wrapper already sanitized and # does not reassign messages before dispatch, so it sets # ``_litellm_messages_presanitized`` to skip this redundant second - # full-messages scan. The same flag marks this call as a second pass for - # cache_control injection: the async wrapper already judged the pristine - # client input, and re-judging after injection would misread litellm's own - # marks as client ones. Pop it so it never leaks into provider params. - presanitized = kwargs.pop("_litellm_messages_presanitized", False) - if not presanitized: + # full-messages scan. Pop it so it never leaks into provider params. + if not kwargs.pop("_litellm_messages_presanitized", False): messages = strip_empty_text_blocks_from_anthropic_messages(messages) messages = sanitize_tool_use_ids_in_anthropic_messages(messages) @@ -435,13 +429,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, - system, - kwargs, - model=model, - custom_llm_provider=custom_llm_provider, - tools=tools, - is_first_pass=not presanitized, + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index f96064c0591..3584297b35f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -522,7 +522,6 @@ async def acompletion( model=model, custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, - is_first_pass=True, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5081,7 +5080,6 @@ def completion( # type: ignore model=model, custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, - is_first_pass=not acompletion, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 601978bb04f..efb189088b6 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -1,6 +1,6 @@ from typing import Literal, Optional, Union -from typing_extensions import TypedDict +from typing_extensions import NotRequired, TypedDict from litellm.types.llms.openai import ChatCompletionCachedContent @@ -12,6 +12,7 @@ class CacheControlMessageInjectionPoint(TypedDict): role: Optional[Literal["user", "system", "assistant"]] # Optional: target by role (user, system, assistant) index: Optional[Union[int, str]] # Optional: target by specific index control: Optional[ChatCompletionCachedContent] + _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran class CacheControlToolConfigInjectionPoint(TypedDict): @@ -19,6 +20,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: Optional[ChatCompletionCachedContent] + _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran CacheControlInjectionPoint = Union[ diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 85e77f2f493..d94f0d5f47e 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1265,8 +1265,11 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point(): ) assert _count_cache_control(processed) == 3 - # The tool_config point is passed through for the provider transform. - assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}] + # The tool_config point is passed through for the provider transform, + # stamped so re-entries never re-judge it against litellm's own marks. + assert non_default_params["cache_control_injection_points"] == [ + {"location": "tool_config", "_litellm_judged": True} + ] @pytest.mark.asyncio @@ -1753,17 +1756,16 @@ class TestConfiguredInjectionPointsStandDown: V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] - def _seed(self, params, messages, tools=None, is_first_pass=True): + def _seed(self, params, messages, tools=None): AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=params, messages=messages, model="claude-sonnet-4-5", custom_llm_provider="anthropic", tools=tools, - is_first_pass=is_first_pass, ) - def _inject(self, messages, kwargs, system="sys", tools=None, is_first_pass=True): + def _inject(self, messages, kwargs, system="sys", tools=None): return AnthropicCacheControlHook.maybe_inject_cache_control( messages, system, @@ -1771,7 +1773,6 @@ class TestConfiguredInjectionPointsStandDown: model="claude-sonnet-4-5", custom_llm_provider="anthropic", tools=tools, - is_first_pass=is_first_pass, ) def test_configured_points_dropped_when_messages_carry_cache_control(self): @@ -1798,13 +1799,13 @@ class TestConfiguredInjectionPointsStandDown: self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES)) assert params["cache_control_injection_points"] is configured - def test_second_pass_keeps_points_despite_injected_marks(self): + def test_judged_remainder_survives_reentry_despite_injected_marks(self): """acompletion() re-enters completion() after injection ran, with only the - non-message points written back; the second pass must not misread litellm's - own marks as client ones and drop that remainder.""" - remainder = [{"location": "tool_config"}] + stamped non-message points written back; the re-entry must not misread + litellm's own marks as client ones and drop that remainder.""" + remainder = [{"location": "tool_config", "_litellm_judged": True}] params = {"cache_control_injection_points": remainder} - self._seed(params, copy.deepcopy(self.MARKED_MESSAGES), is_first_pass=False) + self._seed(params, copy.deepcopy(self.MARKED_MESSAGES)) assert params["cache_control_injection_points"] is remainder def test_v1_messages_stand_down_when_content_block_marked(self): @@ -1841,18 +1842,23 @@ class TestConfiguredInjectionPointsStandDown: _, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] - def test_v1_messages_second_pass_writes_back_remainder(self): - """The async entry re-dispatches into the sync handler after injecting; the - surviving tool_config remainder must survive that second pass even though - the messages now carry litellm's own marks.""" - marked = [ - {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} - ] - remainder = [{"location": "tool_config"}] - kwargs = {"cache_control_injection_points": remainder} - result_msgs, _ = self._inject(copy.deepcopy(marked), kwargs, is_first_pass=False) - assert result_msgs == marked - assert kwargs["cache_control_injection_points"] == remainder + def test_v1_messages_reentry_flow_preserves_tool_config_remainder(self): + """The advisor interceptor re-enters anthropic_messages() with the outer + request's kwargs and post-injection messages. The first pass applies the + message point and writes back a stamped tool_config remainder; the + re-entry must keep that remainder even though the messages and system + now carry litellm's own marks.""" + points = [{"location": "message", "role": "system"}, {"location": "tool_config"}] + kwargs = {"cache_control_injection_points": copy.deepcopy(points)} + msgs1, sys1 = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) + assert sys1[0]["cache_control"] == {"type": "ephemeral"} + expected_remainder = [{"location": "tool_config", "_litellm_judged": True}] + assert kwargs["cache_control_injection_points"] == expected_remainder + + msgs2, sys2 = self._inject(msgs1, kwargs, system=sys1) + assert kwargs["cache_control_injection_points"] == expected_remainder + assert msgs2 == msgs1 + assert sys2 == sys1 class TestAnthropicPromptCachingEnvVars: