diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4f9b18713d0..b06372baa78 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -121,6 +121,12 @@ def _carries_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS) +def _tool_carries_cache_breakpoint(tool: object) -> bool: + return _carries_cache_breakpoint(tool) or ( + isinstance(tool, dict) and _carries_cache_breakpoint(tool.get("function")) + ) + + def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES @@ -131,6 +137,8 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: # rather than spending them on a list that is still missing some of their targets. CARRY_UNMATCHED_MESSAGE_POINTS: Final = "_litellm_carry_unmatched_cache_control_points" +EXTERNAL_BREAKPOINTS_STAMP: Final = "_litellm_external_breakpoints" + class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod @@ -205,10 +213,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: remaining_points.append(point) - # Non-message points (currently Bedrock tool_config) are handled in the - # provider transform, where each tool_config point appends at most one - # cachePoint to the tools. That block also counts toward Anthropic's - # limit, so reserve a slot for it here to leave room. stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") openai_dialect: Final = ( stamped_dialect @@ -233,8 +237,11 @@ class AnthropicCacheControlHook(CustomPromptManagement): if carry_unmatched else tuple(message_points) ) - reserved_blocks: Final = ( - 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 + stamped_external: Final = injection_points[0].get(EXTERNAL_BREAKPOINTS_STAMP) + reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages( + remaining_points, + stamped_external if isinstance(stamped_external, int) else 0, + openai_dialect, ) breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) processed_messages = self._apply_message_injections( @@ -251,14 +258,12 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Points this pass did not place: non-message ones for the provider transform, and # the deferred role-targeted ones. Deferring is what reaches the Responses API's - # `instructions`, which is only a system message once the bridge builds one. The - # judged stamp is what makes it safe: the next pass must not re-judge points - # against messages this pass already marked (see `_should_stand_down`). + # `instructions`, which is only a system message once the bridge builds one. A later + # pass re-applies them safely: a target that already carries a mark is skipped and + # the census counts every mark on the wire, litellm's own included. carried_points: Final[Sequence[CacheControlInjectionPoint]] = (*remaining_points, *carried_message_points) if carried_points: - non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged( - carried_points - ) + non_default_params["cache_control_injection_points"] = list(carried_points) return model, processed_messages, non_default_params @@ -293,6 +298,34 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages) + @staticmethod + def count_external_cache_breakpoints(tools: Iterable[object] | None, cache_control: object = None) -> int: + """Client breakpoints outside messages and system that the provider cap still counts. + + A tool carries its mark at the top level (Anthropic shape) or under ``function`` + (OpenAI shape); the Anthropic chat transform forwards both. A top-level + ``cache_control`` is Anthropic's automatic caching, which places one breakpoint + of its own on top of the explicit ones. + """ + automatic_blocks: Final = 1 if cache_control is not None else 0 + tool_blocks: Final = sum(1 for tool in tools if _tool_carries_cache_breakpoint(tool)) if tools else 0 + return automatic_blocks + tool_blocks + + @staticmethod + def _blocks_reserved_outside_messages( + remaining_points: Sequence[CacheControlInjectionPoint], external_breakpoints: int, openai_dialect: bool + ) -> int: + """Slots of the provider cap that the message census cannot see. + + The client's breakpoints on tools and its automatic top-level ``cache_control`` + are already on the wire, and a ``tool_config`` point becomes one more cachePoint + in the Bedrock converse transform. OpenAI's cap counts only its own block markers. + """ + if openai_dialect: + return 0 + tool_config_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 + return external_breakpoints + tool_config_blocks + @staticmethod def _apply_message_injections( points: Sequence[CacheControlMessageInjectionPoint], @@ -473,11 +506,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): def apply_to_anthropic_messages_request( messages: list[dict], system: str | list | None, - injection_points: list[CacheControlInjectionPoint], + injection_points: Sequence[CacheControlInjectionPoint], openai_dialect: bool = False, + external_breakpoints: int = 0, ) -> tuple[list[dict], str | list | None, list[CacheControlInjectionPoint]]: """Apply cache control injection for the Anthropic-native v1/messages endpoint. + ``external_breakpoints`` is the client's breakpoint count outside ``messages`` and + ``system`` (see ``count_external_cache_breakpoints``); it shrinks the budget so + the request never exceeds the provider cap. + Returns (messages, system, remaining_non_message_points). """ if not injection_points: @@ -500,8 +538,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: remaining_points.append(point) - reserved_blocks: Final = ( - 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 + reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages( + remaining_points, external_breakpoints, openai_dialect ) max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks @@ -556,30 +594,26 @@ class AnthropicCacheControlHook(CustomPromptManagement): return ChatCompletionCachedContent(type="ephemeral") @staticmethod - def _stamped_as_judged(points: Sequence[CacheControlInjectionPoint]) -> Sequence[Mapping[str, object]]: - """Mark written-back points as having passed the client cache_control judgment. - - Builds copies because config-owned point dicts are shared across - requests; mutating them would leak the stamp into future requests. - """ - return AnthropicCacheControlHook._stamped(points, "_litellm_judged", True) - - @staticmethod - def _judged_configured_points( + def _stamped_for_prompt_hook( points: Sequence[CacheControlInjectionPoint], - messages: list[AllMessageValues], - tools: list[object] | None, - cache_control: object, + external_breakpoints: int, model: str, custom_llm_provider: str | None, api_base: object, prompt_cache_options: object, - ) -> Sequence[Mapping[str, object]] | None: - if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools, cache_control): - return None - return AnthropicCacheControlHook._stamped_with_dialect( + ) -> Sequence[Mapping[str, object]]: + """Carry onto the points what the prompt-management hook never receives. + + The hook sees neither the tools nor the request kwargs, so the target dialect + and the client's breakpoint count outside the message list ride on the points. + Builds copies because config-owned point dicts are shared across requests. + """ + with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect( points, model, custom_llm_provider, api_base, prompt_cache_options ) + if external_breakpoints == 0: + return with_dialect + return AnthropicCacheControlHook._stamped(with_dialect, EXTERNAL_BREAKPOINTS_STAMP, external_breakpoints) @staticmethod def _stamped_with_dialect( @@ -600,32 +634,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) @staticmethod - def _stamped( - points: Sequence[CacheControlInjectionPoint], key: str, value: object - ) -> Sequence[Mapping[str, object]]: + def _stamped(points: Sequence[Mapping[str, object]], key: str, value: object) -> Sequence[Mapping[str, object]]: return [{**point, key: value} for point in points] - @staticmethod - def _should_stand_down( - points: Sequence[CacheControlInjectionPoint], - messages: list[AllMessageValues], - system: str | list | None, - tools: list | None, - cache_control: object = None, - ) -> bool: - """Whether configured injection points must yield to client-set cache_control. - - Points that a prior pass over this request already judged and wrote - back carry the internal judged stamp; any re-entry (acompletion - re-entering completion, the async-to-sync /v1/messages dispatch, - interceptor sub-calls reusing the request kwargs) must not re-judge - them, because by then the messages carry litellm's own injected marks - and the judgment would misread those as client breakpoints. - """ - if all(point.get("_litellm_judged") for point in points): - return False - return AnthropicCacheControlHook._request_has_cache_control(messages, system, tools, cache_control) - @staticmethod def _request_has_cache_control( messages: list[AllMessageValues], @@ -635,28 +646,15 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> bool: """Return True if the request already carries any client-supplied cache_control. - When the client (e.g. Claude Code) already marks its own breakpoints we - stand down entirely rather than add more, per the auto-caching contract. - Tools count: they are a breakpoint the client can mark, they count toward - the provider's four-block limit, and caching only the tool definitions is - a common pattern, so injecting alongside them can exceed the cap. Tools - carry the mark either at the top level (Anthropic shape) or nested under - ``function`` (OpenAI shape); the Anthropic chat transform accepts both. + Only the automatic defaults stand down on it: a client that marks its own + breakpoints (Claude Code does) has a caching strategy the defaults would + clash with. Configured injection points are an explicit instruction and are + applied alongside the client's marks, bounded by the provider cap. """ - if cache_control is not None: - return True - if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0: - return True - if tools is not None: - return any( - isinstance(tool, dict) - and ( - tool.get("cache_control") is not None - or (isinstance(tool.get("function"), dict) and tool["function"].get("cache_control") is not None) - ) - for tool in tools - ) - return False + return ( + AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) + + AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control) + ) > 0 @staticmethod def get_default_injection_points( @@ -779,31 +777,25 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> None: """For /chat/completions: resolve the injection points the request should carry. - Configured injection points win over the automatic defaults, but stand - down entirely when the client already marked its own cache_control - breakpoints (messages or tools): injecting alongside them clashes with - the client's caching strategy and can exceed the provider's four-block - limit. The judgment happens once per request; points a prior pass - wrote back carry the judged stamp and are never re-judged (see - ``_should_stand_down``). Seeding the param lets the existing - prompt-management gate and the AnthropicCacheControlHook run - unchanged. + Configured injection points win over the automatic defaults and are applied + even when the client marked its own cache_control elsewhere in the request; + the provider's four-block cap bounds them, counting the client's marks on + messages, tools and the top-level ``cache_control``. Only the defaults stand + down on client marks. Seeding the param lets the existing prompt-management + gate and the AnthropicCacheControlHook run unchanged. """ - if non_default_params.get("cache_control_injection_points"): - judged: Final = AnthropicCacheControlHook._judged_configured_points( - non_default_params["cache_control_injection_points"], - messages, - tools, - non_default_params.get("cache_control"), + configured: Final = non_default_params.get("cache_control_injection_points") + if configured: + non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_for_prompt_hook( + configured, + AnthropicCacheControlHook.count_external_cache_breakpoints( + tools, non_default_params.get("cache_control") + ), model, custom_llm_provider, api_base, non_default_params.get("prompt_cache_options"), ) - if judged is None: - non_default_params.pop("cache_control_injection_points") - else: - non_default_params["cache_control_injection_points"] = judged return points: Final = AnthropicCacheControlHook.get_default_injection_points( messages=messages, @@ -904,15 +896,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> tuple[list[dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. - Configured points stand down entirely when the client already marked - its own cache_control breakpoints anywhere in the request. The - judgment happens once per request; points a prior pass wrote back - carry the judged stamp and are never re-judged (see - ``_should_stand_down``). When none are configured but + Configured points are applied even when the client marked its own + cache_control elsewhere in the request, bounded by the provider cap, + which counts the client's marks on messages, system, tools and the + top-level ``cache_control``. When none are configured but ``litellm.enable_anthropic_prompt_caching`` or the per-request ``enable_prompt_caching`` kwarg (stamped from key metadata) is on, - synthesize default breakpoints for the native /v1/messages path. Pops - both keys from kwargs; + synthesize default breakpoints for the native /v1/messages path; those + defaults alone stand down on client marks. Pops both keys from kwargs; if remaining (non-message) points exist they are written back so downstream transforms can handle them. """ @@ -924,13 +915,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): configured: Final = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) - if configured and AnthropicCacheControlHook._should_stand_down( - configured, typed_messages, system, tools, cache_control - ): - return messages, system - injection_points: list[CacheControlInjectionPoint] = configured or [] - if not injection_points and model is not None: - injection_points = AnthropicCacheControlHook.get_default_injection_points( + injection_points: Final[Sequence[CacheControlInjectionPoint]] = configured or ( + AnthropicCacheControlHook.get_default_injection_points( messages=typed_messages, system=system, tools=tools, @@ -940,6 +926,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): cache_control=cache_control, request_kwargs=kwargs, ) + if model is not None + else () + ) if not injection_points: return messages, system @@ -952,6 +941,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): system=system, injection_points=injection_points, openai_dialect=openai_dialect, + external_breakpoints=AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control), ) breakpoints_added: Final = ( AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before @@ -960,7 +950,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if openai_dialect and breakpoints_added > 0: kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) if remaining: - kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining) + kwargs["cache_control_injection_points"] = remaining return messages, system @property diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index ef414f22c3b..20e7885a2bf 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -17,8 +17,8 @@ class CacheControlMessageInjectionPoint(TypedDict): role: Literal["user", "system", "assistant"] | None # Optional: target by role (user, system, assistant) index: int | str | None # Optional: target by specific index control: ChatCompletionCachedContent | None - _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran _litellm_openai_dialect: NotRequired[ReadOnly[bool]] + _litellm_external_breakpoints: NotRequired[ReadOnly[int]] class CacheControlToolConfigInjectionPoint(TypedDict): @@ -26,8 +26,8 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: ChatCompletionCachedContent | None - _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran _litellm_openai_dialect: NotRequired[ReadOnly[bool]] + _litellm_external_breakpoints: NotRequired[ReadOnly[int]] CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 92b1185e542..3424cc5fed6 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1276,11 +1276,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point(): ) assert _count_cache_control(processed) == 3 - # The tool_config point is passed through for the provider transform, - # stamped so re-entries never re-judge it against litellm's own marks. - assert non_default_params["cache_control_injection_points"] == [ - {"location": "tool_config", "_litellm_judged": True} - ] + assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}] @pytest.mark.asyncio @@ -2085,13 +2081,17 @@ class TestPerKeyEnablePromptCaching: assert result_msgs == messages -class TestConfiguredInjectionPointsStandDown: - """Configured cache_control_injection_points must stand down entirely when the - client already set its own cache_control anywhere in the request (LIT-4582); - injecting alongside client breakpoints clashes with the client's caching - strategy and can push the request past Anthropic's four-block limit.""" +class TestConfiguredInjectionPointsSurviveClientMarks: + """Configured cache_control_injection_points are an explicit instruction, so they + apply alongside the client's own cache_control marks (LIT-7586, #40675) instead of + standing down on them. What bounds them is Anthropic's four-block cap, which has to + count the client's marks on messages, system, tools and the root ``cache_control`` + (LIT-4582: a client-marked tool the cap could not see produced "Found 5" 400s). + Only the automatic defaults stand down on client marks.""" CONFIGURED = [{"location": "message", "role": "system"}] + TAIL_POINT = [{"location": "message", "index": -1}] + EPHEMERAL = {"type": "ephemeral"} CLEAN_MESSAGES: List[AllMessageValues] = [ {"role": "system", "content": "sys"}, @@ -2105,6 +2105,23 @@ class TestConfiguredInjectionPointsStandDown: V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + MARKED_TOOL_TOP_LEVEL = { + "type": "function", + "function": {"name": "t", "parameters": {}}, + "cache_control": {"type": "ephemeral"}, + } + MARKED_TOOL_NESTED = {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}} + UNMARKED_TOOL = {"type": "function", "function": {"name": "t", "parameters": {}}} + MARKED_V1_TOOL = {"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}} + UNMARKED_V1_TOOL = {"name": "t", "input_schema": {}} + + @staticmethod + def _marked_user_turns(count): + return [ + {"role": "user", "content": [{"type": "text", "text": f"turn {i}", "cache_control": {"type": "ephemeral"}}]} + for i in range(count) + ] + def _seed(self, params, messages, tools=None): AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=params, @@ -2114,6 +2131,17 @@ class TestConfiguredInjectionPointsStandDown: tools=tools, ) + def _chat(self, params, messages): + _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model="claude-sonnet-4-5", + messages=messages, + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return processed + def _inject(self, messages, kwargs, system="sys", tools=None): return AnthropicCacheControlHook.maybe_inject_cache_control( messages, @@ -2124,23 +2152,64 @@ class TestConfiguredInjectionPointsStandDown: tools=tools, ) - def test_configured_points_dropped_when_messages_carry_cache_control(self): + def test_chat_tail_point_applies_when_client_marked_the_system_block(self): + """The issue's shape: the client caches its system prompt, the deployment is + configured to cache the trailing turn, and both marks must reach the provider.""" + messages: List[AllMessageValues] = [ + {"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "history"}, + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "question"}, + ] + params = {"cache_control_injection_points": copy.deepcopy(self.TAIL_POINT)} + self._seed(params, messages) + processed = self._chat(params, messages) + assert processed[0] == messages[0] + assert processed[-1] == {"role": "user", "content": "question", "cache_control": self.EPHEMERAL} + assert _count_cache_control(processed) == 2 + + def test_chat_configured_points_apply_when_messages_carry_cache_control(self): params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} self._seed(params, copy.deepcopy(self.MARKED_MESSAGES)) - assert "cache_control_injection_points" not in params + processed = self._chat(params, copy.deepcopy(self.MARKED_MESSAGES)) + assert processed[0] == {"role": "system", "content": "sys", "cache_control": self.EPHEMERAL} + assert processed[1] == self.MARKED_MESSAGES[1] @pytest.mark.parametrize( - "tool", - [ - {"type": "function", "function": {"name": "t", "parameters": {}}, "cache_control": {"type": "ephemeral"}}, - {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}, - ], - ids=["top_level", "nested_in_function"], + "tool", [MARKED_TOOL_TOP_LEVEL, MARKED_TOOL_NESTED], ids=["top_level", "nested_in_function"] ) - def test_configured_points_dropped_when_tools_carry_cache_control(self, tool): + def test_chat_configured_points_apply_when_tools_carry_cache_control(self, tool): params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES), tools=[tool]) - assert "cache_control_injection_points" not in params + processed = self._chat(params, copy.deepcopy(self.CLEAN_MESSAGES)) + assert processed[0] == {"role": "system", "content": "sys", "cache_control": self.EPHEMERAL} + + @pytest.mark.parametrize( + "tool,injected", + [(MARKED_TOOL_TOP_LEVEL, 0), (MARKED_TOOL_NESTED, 0), (UNMARKED_TOOL, 1)], + ids=["marked_top_level", "marked_nested_in_function", "unmarked"], + ) + def test_chat_cap_counts_client_marked_tools(self, tool, injected): + """LIT-4582 regression: the prompt-management hook never sees the tools, so the + seeding pass has to carry the client's tool marks into the cap or a configured + point lands as a fifth block.""" + messages = [{"role": "system", "content": "sys"}, *self._marked_user_turns(3)] + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + self._seed(params, copy.deepcopy(messages), tools=[tool]) + processed = self._chat(params, copy.deepcopy(messages)) + assert _count_cache_control(processed) == 3 + injected + + @pytest.mark.parametrize("marked_turns,injected", [(2, 1), (3, 0)]) + def test_chat_root_cache_control_reserves_a_slot(self, marked_turns, injected): + """Anthropic's automatic caching (a top-level ``cache_control``) places one + breakpoint of its own, so it counts toward the cap like a client mark.""" + messages = [{"role": "system", "content": "sys"}, *self._marked_user_turns(marked_turns)] + root_cache_control = {"type": "ephemeral"} + params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED), "cache_control": root_cache_control} + self._seed(params, copy.deepcopy(messages)) + processed = self._chat(params, copy.deepcopy(messages)) + assert _count_cache_control(processed) == marked_turns + injected + assert params["cache_control"] is root_cache_control def test_configured_points_kept_when_request_is_unmarked(self): configured = copy.deepcopy(self.CONFIGURED) @@ -2148,60 +2217,68 @@ class TestConfiguredInjectionPointsStandDown: self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES)) assert params["cache_control_injection_points"] is configured - def test_judged_remainder_survives_reentry_despite_injected_marks(self): - """acompletion() re-enters completion() after injection ran, with only the - stamped non-message points written back; the re-entry must not misread - litellm's own marks as client ones and drop that remainder.""" - remainder = [{"location": "tool_config", "_litellm_judged": True}] - params = {"cache_control_injection_points": remainder} - self._seed(params, copy.deepcopy(self.MARKED_MESSAGES)) - assert params["cache_control_injection_points"] is remainder + def test_chat_reentry_over_injected_messages_adds_no_duplicate_marks(self): + """acompletion() re-enters completion() and interceptor sub-calls reuse the + request kwargs, so the same configured points meet messages that already carry + litellm's own marks; the second pass must leave them as they are.""" + points = [{"location": "message", "role": "system"}, {"location": "tool_config"}] + first_params = {"cache_control_injection_points": copy.deepcopy(points)} + self._seed(first_params, copy.deepcopy(self.MARKED_MESSAGES)) + first = self._chat(first_params, copy.deepcopy(self.MARKED_MESSAGES)) + assert _count_cache_control(first) == 2 + assert first_params["cache_control_injection_points"] == [{"location": "tool_config"}] - def test_v1_messages_stand_down_when_content_block_marked(self): + second_params = {"cache_control_injection_points": copy.deepcopy(points)} + self._seed(second_params, copy.deepcopy(first)) + second = self._chat(second_params, copy.deepcopy(first)) + assert second == first + assert second_params["cache_control_injection_points"] == [{"location": "tool_config"}] + + def test_v1_messages_configured_point_applies_when_content_block_marked(self): messages = [ {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} ] kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} result_msgs, result_sys = self._inject(copy.deepcopy(messages), kwargs) assert result_msgs == messages - assert result_sys == "sys" + assert result_sys == [{"type": "text", "text": "sys", "cache_control": self.EPHEMERAL}] assert "cache_control_injection_points" not in kwargs - def test_v1_messages_stand_down_when_system_block_marked(self): - """A configured point targeting a message must not fire when the client - marked the system prompt; the old behavior injected into the message - because only the exact targeted position was guarded.""" + def test_v1_messages_tail_point_applies_when_system_block_marked(self): system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] - kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + kwargs = {"cache_control_injection_points": copy.deepcopy(self.TAIL_POINT)} result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, system=system) - assert result_msgs == self.V1_MESSAGES + assert result_msgs == [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": self.EPHEMERAL}]}] assert result_sys == system - assert "cache_control_injection_points" not in kwargs - def test_v1_messages_stand_down_when_tools_marked(self): - tools = [{"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}}] + def test_v1_messages_configured_point_applies_when_tools_marked(self): kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} - result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=tools) + result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=[self.MARKED_V1_TOOL]) assert result_msgs == self.V1_MESSAGES - assert result_sys == "sys" - assert "cache_control_injection_points" not in kwargs + assert result_sys == [{"type": "text", "text": "sys", "cache_control": self.EPHEMERAL}] + + @pytest.mark.parametrize( + "tool,expected_system", + [ + (MARKED_V1_TOOL, "sys"), + (UNMARKED_V1_TOOL, [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), + ], + ids=["marked", "unmarked"], + ) + def test_v1_messages_cap_counts_client_marked_tools(self, tool, expected_system): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} + _, result_sys = self._inject(self._marked_user_turns(3), kwargs, tools=[tool]) + assert result_sys == expected_system def test_v1_messages_configured_points_apply_when_unmarked(self): kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)} _, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] - @pytest.mark.parametrize( - "configured", - [None, CONFIGURED], - ids=["automatic_defaults", "configured_points"], - ) - def test_v1_messages_stands_down_for_root_cache_control(self, monkeypatch, configured): + def test_v1_messages_automatic_defaults_stand_down_for_root_cache_control(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) root_cache_control = {"type": "ephemeral"} kwargs = {"cache_control": root_cache_control, "litellm_metadata": {}} - if configured is not None: - kwargs["cache_control_injection_points"] = copy.deepcopy(configured) result_messages, result_system = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) @@ -2210,17 +2287,33 @@ class TestConfiguredInjectionPointsStandDown: assert kwargs["cache_control"] is root_cache_control assert "litellm_gateway_injected_cache" not in kwargs["litellm_metadata"] + @pytest.mark.parametrize( + "marked_turns,expected_system", + [(2, [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), (3, "sys")], + ) + def test_v1_messages_configured_points_apply_with_root_cache_control_reserving_a_slot( + self, marked_turns, expected_system + ): + root_cache_control = {"type": "ephemeral"} + kwargs = { + "cache_control": root_cache_control, + "cache_control_injection_points": copy.deepcopy(self.CONFIGURED), + } + _, result_system = self._inject(self._marked_user_turns(marked_turns), kwargs) + assert result_system == expected_system + assert kwargs["cache_control"] is root_cache_control + def test_v1_messages_reentry_flow_preserves_tool_config_remainder(self): """The advisor interceptor re-enters anthropic_messages() with the outer request's kwargs and post-injection messages. The first pass applies the - message point and writes back a stamped tool_config remainder; the - re-entry must keep that remainder even though the messages and system - now carry litellm's own marks.""" + message point and writes back the tool_config remainder; the re-entry must + keep that remainder and add no mark even though the messages and system + now carry litellm's own.""" points = [{"location": "message", "role": "system"}, {"location": "tool_config"}] kwargs = {"cache_control_injection_points": copy.deepcopy(points)} msgs1, sys1 = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs) assert sys1[0]["cache_control"] == {"type": "ephemeral"} - expected_remainder = [{"location": "tool_config", "_litellm_judged": True}] + expected_remainder = [{"location": "tool_config"}] assert kwargs["cache_control_injection_points"] == expected_remainder msgs2, sys2 = self._inject(msgs1, kwargs, system=sys1) @@ -2459,22 +2552,22 @@ class TestOpenAIPromptCacheBreakpoint: assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] assert kwargs == {} - def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self): + def test_v1_messages_configured_points_apply_beside_client_content_breakpoint(self): messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}] kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} result, system = self._inject(messages, "sys", kwargs) assert result == messages - assert system == "sys" - assert kwargs == {} + assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + assert kwargs == {"prompt_cache_options": self.EXPLICIT} - def test_v1_messages_client_system_breakpoint_makes_configured_points_stand_down(self): + def test_v1_messages_tail_point_applies_beside_client_system_breakpoint(self): system = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} result, result_system = self._inject(messages, system, kwargs) - assert result == messages + assert result == [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}] assert result_system == system - assert kwargs == {} + assert kwargs == {"prompt_cache_options": self.EXPLICIT} def test_chat_system_string_wrapped_with_block_breakpoint(self): params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} @@ -2538,18 +2631,25 @@ class TestOpenAIPromptCacheBreakpoint: assert processed[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} assert params == {} - def test_chat_client_breakpoint_makes_seeded_points_stand_down(self): + def test_chat_seeded_points_apply_beside_client_breakpoint(self): params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}, + ] AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=params, - messages=[ - {"role": "system", "content": "sys"}, - {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}, - ], + messages=messages, model="openai/gpt-5.6", custom_llm_provider="openai", ) - assert params == {} + assert params["cache_control_injection_points"] == [ + {"location": "message", "role": "system", "_litellm_openai_dialect": True} + ] + _, processed, _ = self._chat(messages, params) + assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + assert processed[1] == messages[1] + assert params["prompt_cache_options"] == self.EXPLICIT def test_cap_counts_client_breakpoints_of_both_kinds(self): messages = [ @@ -3143,7 +3243,7 @@ class TestRecordGatewayInjection: assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT def test_configured_points_skipping_a_marked_target_record_nothing(self): - """Configured injection stands down on client breakpoints, so no marker lands.""" + """A configured point whose target the client already marked places nothing, so no marker lands.""" kwargs: dict = { "litellm_metadata": {}, "cache_control_injection_points": [{"location": "message", "role": "system", "index": None}],