From e0648571ed9dc6d31337e975f0a55c36a77d985d Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 18 Jul 2026 17:11:03 -0700 Subject: [PATCH] fix(anthropic): carry the stand-down judgment inside written-back injection points --- .../anthropic_cache_control_hook.py | 67 +++++++++++++------ .../messages/handler.py | 24 ++----- litellm/main.py | 2 - .../anthropic_cache_control_hook.py | 4 +- .../test_anthropic_cache_control_hook.py | 52 +++++++------- 5 files changed, 84 insertions(+), 65 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 54f0732fdf1..faedf8ae1a3 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -91,7 +91,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Pass through non-message injection points for provider-specific handling if remaining_points: - non_default_params["cache_control_injection_points"] = remaining_points + non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged( + remaining_points + ) return model, processed_messages, non_default_params @@ -310,6 +312,35 @@ class AnthropicCacheControlHook(CustomPromptManagement): return ChatCompletionCachedContent(type="ephemeral", ttl=ttl) return ChatCompletionCachedContent(type="ephemeral") + @staticmethod + def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]: + """Mark written-back points as having passed the client cache_control judgment. + + Builds copies because config-owned point dicts are shared across + requests; mutating them would leak the stamp into future requests. + """ + return [{**point, "_litellm_judged": True} for point in points] + + @staticmethod + def _should_stand_down( + points: list[CacheControlInjectionPoint], + messages: list[AllMessageValues], + system: str | list | None, + tools: list | None, + ) -> bool: + """Whether configured injection points must yield to client-set cache_control. + + Points that a prior pass over this request already judged and wrote + back carry the internal judged stamp; any re-entry (acompletion + re-entering completion, the async-to-sync /v1/messages dispatch, + interceptor sub-calls reusing the request kwargs) must not re-judge + them, because by then the messages carry litellm's own injected marks + and the judgment would misread those as client breakpoints. + """ + if all(point.get("_litellm_judged") for point in points): + return False + return AnthropicCacheControlHook._request_has_cache_control(messages, system, tools) + @staticmethod def _request_has_cache_control( messages: list[AllMessageValues], @@ -400,7 +431,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, - is_first_pass: bool = True, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -408,15 +438,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): down entirely when the client already marked its own cache_control breakpoints (messages or tools): injecting alongside them clashes with the client's caching strategy and can exceed the provider's four-block - limit. Only the first pass over a request may make that judgment; - ``acompletion`` re-enters ``completion`` after injection has already - run, and a later pass would mistake litellm's own injected marks for - client ones and drop the non-message points reserved for provider - transforms. Seeding the param lets the existing prompt-management gate - and the AnthropicCacheControlHook run unchanged. + limit. The judgment happens once per request; points a prior pass + wrote back carry the judged stamp and are never re-judged (see + ``_should_stand_down``). Seeding the param lets the existing + prompt-management gate and the AnthropicCacheControlHook run + unchanged. """ if non_default_params.get("cache_control_injection_points"): - if is_first_pass and AnthropicCacheControlHook._request_has_cache_control(messages, None, tools): + if AnthropicCacheControlHook._should_stand_down( + non_default_params["cache_control_injection_points"], messages, None, tools + ): non_default_params.pop("cache_control_injection_points") return points = AnthropicCacheControlHook.get_default_injection_points( @@ -437,16 +468,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, - is_first_pass: bool = True, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. Configured points stand down entirely when the client already marked - its own cache_control breakpoints anywhere in the request, judged only - on the first pass: the async entry re-dispatches into the sync handler - after injection has run, and a later pass would mistake litellm's own - injected marks for client ones and drop the written-back non-message - points. When none are configured but + its own cache_control breakpoints anywhere in the request. The + judgment happens once per request; points a prior pass wrote back + carry the judged stamp and are never re-judged (see + ``_should_stand_down``). When none are configured but ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default breakpoints for the native /v1/messages path. Pops the key from kwargs; if remaining (non-message) points exist they are written back so @@ -456,11 +485,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) - if ( - configured - and is_first_pass - and AnthropicCacheControlHook._request_has_cache_control(typed_messages, system, tools) - ): + if configured and AnthropicCacheControlHook._should_stand_down(configured, typed_messages, system, tools): return messages, system injection_points: list[CacheControlInjectionPoint] = configured or [] if not injection_points and model is not None: @@ -480,7 +505,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): injection_points=injection_points, ) if remaining: - kwargs["cache_control_injection_points"] = remaining + kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining) return messages, system @property diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 92151b45b75..703ccf13c27 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -351,11 +351,9 @@ async def anthropic_messages( custom_llm_provider=custom_llm_provider, # messages were already empty-text-block sanitized at the top of this # function and are NOT reassigned before this dispatch, so the handler - # can skip its (otherwise redundant) second full-messages scan. It also - # tells the handler that cache_control injection already judged the - # pristine client input here. Passed explicitly (not via **kwargs) so - # it only affects this direct dispatch -- interceptor / sync entry - # points still sanitize. + # can skip its (otherwise redundant) second full-messages scan. Passed + # explicitly (not via **kwargs) so it only affects this direct + # dispatch -- interceptor / sync entry points still sanitize. _litellm_messages_presanitized=True, **kwargs, ) @@ -421,12 +419,8 @@ def anthropic_messages_handler( # protection as the async wrapper. The async wrapper already sanitized and # does not reassign messages before dispatch, so it sets # ``_litellm_messages_presanitized`` to skip this redundant second - # full-messages scan. The same flag marks this call as a second pass for - # cache_control injection: the async wrapper already judged the pristine - # client input, and re-judging after injection would misread litellm's own - # marks as client ones. Pop it so it never leaks into provider params. - presanitized = kwargs.pop("_litellm_messages_presanitized", False) - if not presanitized: + # full-messages scan. Pop it so it never leaks into provider params. + if not kwargs.pop("_litellm_messages_presanitized", False): messages = strip_empty_text_blocks_from_anthropic_messages(messages) messages = sanitize_tool_use_ids_in_anthropic_messages(messages) @@ -435,13 +429,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, - system, - kwargs, - model=model, - custom_llm_provider=custom_llm_provider, - tools=tools, - is_first_pass=not presanitized, + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index f96064c0591..3584297b35f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -522,7 +522,6 @@ async def acompletion( model=model, custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, - is_first_pass=True, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5081,7 +5080,6 @@ def completion( # type: ignore model=model, custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, - is_first_pass=not acompletion, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 601978bb04f..efb189088b6 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -1,6 +1,6 @@ from typing import Literal, Optional, Union -from typing_extensions import TypedDict +from typing_extensions import NotRequired, TypedDict from litellm.types.llms.openai import ChatCompletionCachedContent @@ -12,6 +12,7 @@ class CacheControlMessageInjectionPoint(TypedDict): role: Optional[Literal["user", "system", "assistant"]] # Optional: target by role (user, system, assistant) index: Optional[Union[int, str]] # Optional: target by specific index control: Optional[ChatCompletionCachedContent] + _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran class CacheControlToolConfigInjectionPoint(TypedDict): @@ -19,6 +20,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: Optional[ChatCompletionCachedContent] + _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran CacheControlInjectionPoint = Union[ diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 85e77f2f493..d94f0d5f47e 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1265,8 +1265,11 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point(): ) assert _count_cache_control(processed) == 3 - # The tool_config point is passed through for the provider transform. - assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}] + # The tool_config point is passed through for the provider transform, + # stamped so re-entries never re-judge it against litellm's own marks. + assert non_default_params["cache_control_injection_points"] == [ + {"location": "tool_config", "_litellm_judged": True} + ] @pytest.mark.asyncio @@ -1753,17 +1756,16 @@ class TestConfiguredInjectionPointsStandDown: V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] - def _seed(self, params, messages, tools=None, is_first_pass=True): + def _seed(self, params, messages, tools=None): AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=params, messages=messages, model="claude-sonnet-4-5", custom_llm_provider="anthropic", tools=tools, - is_first_pass=is_first_pass, ) - def _inject(self, messages, kwargs, system="sys", tools=None, is_first_pass=True): + def _inject(self, messages, kwargs, system="sys", tools=None): return AnthropicCacheControlHook.maybe_inject_cache_control( messages, system, @@ -1771,7 +1773,6 @@ class TestConfiguredInjectionPointsStandDown: model="claude-sonnet-4-5", custom_llm_provider="anthropic", tools=tools, - is_first_pass=is_first_pass, ) def test_configured_points_dropped_when_messages_carry_cache_control(self): @@ -1798,13 +1799,13 @@ class TestConfiguredInjectionPointsStandDown: self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES)) assert params["cache_control_injection_points"] is configured - def test_second_pass_keeps_points_despite_injected_marks(self): + def test_judged_remainder_survives_reentry_despite_injected_marks(self): """acompletion() re-enters completion() after injection ran, with only the - non-message points written back; the second pass must not misread litellm's - own marks as client ones and drop that remainder.""" - remainder = [{"location": "tool_config"}] + stamped non-message points written back; the re-entry must not misread + litellm's own marks as client ones and drop that remainder.""" + remainder = [{"location": "tool_config", "_litellm_judged": True}] params = {"cache_control_injection_points": remainder} - self._seed(params, copy.deepcopy(self.MARKED_MESSAGES), is_first_pass=False) + self._seed(params, copy.deepcopy(self.MARKED_MESSAGES)) assert params["cache_control_injection_points"] is remainder def test_v1_messages_stand_down_when_content_block_marked(self): @@ -1841,18 +1842,23 @@ class TestConfiguredInjectionPointsStandDown: _, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] - def test_v1_messages_second_pass_writes_back_remainder(self): - """The async entry re-dispatches into the sync handler after injecting; the - surviving tool_config remainder must survive that second pass even though - the messages now carry litellm's own marks.""" - marked = [ - {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} - ] - remainder = [{"location": "tool_config"}] - kwargs = {"cache_control_injection_points": remainder} - result_msgs, _ = self._inject(copy.deepcopy(marked), kwargs, is_first_pass=False) - assert result_msgs == marked - assert kwargs["cache_control_injection_points"] == remainder + def test_v1_messages_reentry_flow_preserves_tool_config_remainder(self): + """The advisor interceptor re-enters anthropic_messages() with the outer + request's kwargs and post-injection messages. The first pass applies the + message point and writes back a stamped tool_config remainder; the + re-entry must keep that remainder even though the messages and system + now carry litellm's own marks.""" + points = [{"location": "message", "role": "system"}, {"location": "tool_config"}] + kwargs = {"cache_control_injection_points": copy.deepcopy(points)} + msgs1, sys1 = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) + assert sys1[0]["cache_control"] == {"type": "ephemeral"} + expected_remainder = [{"location": "tool_config", "_litellm_judged": True}] + assert kwargs["cache_control_injection_points"] == expected_remainder + + msgs2, sys2 = self._inject(msgs1, kwargs, system=sys1) + assert kwargs["cache_control_injection_points"] == expected_remainder + assert msgs2 == msgs1 + assert sys2 == sys1 class TestAnthropicPromptCachingEnvVars: