diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index a7bf9e985f8..9352a952ef7 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -223,8 +223,9 @@ def _as_object_list(value: object | None) -> list[object] | None: return _validated_object_list(value) -def _tool_call_carries_cache_breakpoint(tool_call: object, message: object) -> bool: - if not _carries_cache_breakpoint(tool_call): +def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, openai_dialect: bool = False) -> bool: + keys: Final = ("cache_control",) if openai_dialect else CACHE_BREAKPOINT_KEYS + if all(_attribute_or_key(tool_call, key) is None for key in keys): return False return not tool_call_is_rebuilt_as_server_tool_use( @@ -359,7 +360,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages( remaining_points, external_breakpoints, openai_dialect ) - breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) + breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints( + processed_messages, openai_dialect=openai_dialect + ) processed_messages = self._apply_message_injections( points=applied_message_points, messages=processed_messages, @@ -368,7 +371,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) if ( openai_dialect - and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before + and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages, openai_dialect=True) + > breakpoints_before and non_default_params.get("prompt_cache_options") is None ): non_default_params["prompt_cache_options"] = PromptCacheOptions(mode="implicit") @@ -381,7 +385,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): carried_points: Final[Sequence[CacheControlInjectionPoint]] = ( *AnthropicCacheControlHook._points_with_a_slot_left( remaining_points, - AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) + external_breakpoints, + AnthropicCacheControlHook.count_request_cache_breakpoints( + processed_messages, openai_dialect=openai_dialect + ) + + external_breakpoints, openai_dialect, ), *carried_message_points, @@ -416,11 +423,15 @@ class AnthropicCacheControlHook(CustomPromptManagement): return _served_provider_for_model(model) @staticmethod - def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int: + def count_request_cache_breakpoints( + messages: Iterable[object], system: object = None, openai_dialect: bool = False + ) -> int: system_blocks: Final = ( sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0 ) - return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages) + return system_blocks + sum( + AnthropicCacheControlHook._count_cache_control_blocks(msg, openai_dialect) for msg in messages + ) @staticmethod def count_external_cache_breakpoints( @@ -504,7 +515,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ``max_blocks`` is reached. Injection points are honored in config order, so earlier points win when slots are scarce. """ - used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages) + used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, openai_dialect=openai_dialect) limit_reached = False for point in points: @@ -521,14 +532,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): limit_reached = True break - if AnthropicCacheControlHook._message_has_cache_control(messages[target_index]): + if AnthropicCacheControlHook._message_has_cache_control(messages[target_index], openai_dialect): # Client already marked this message; don't overwrite it. continue messages[target_index] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( messages[target_index], control, openai_dialect ) - if AnthropicCacheControlHook._message_has_cache_control(messages[target_index]): + if AnthropicCacheControlHook._message_has_cache_control(messages[target_index], openai_dialect): used_blocks += 1 if limit_reached: @@ -582,22 +593,24 @@ class AnthropicCacheControlHook(CustomPromptManagement): return [] @staticmethod - def _count_cache_control_blocks(message: object) -> int: + def _count_cache_control_blocks(message: object, openai_dialect: bool = False) -> int: message_count: Final = 1 if _carries_cache_breakpoint(message) else 0 content: Final = _as_object_list(_attribute_or_key(message, "content")) content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0 tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) tool_call_count: Final = ( - sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, message)) + sum( + 1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, message, openai_dialect) + ) if tool_calls else 0 ) return message_count + content_count + tool_call_count @staticmethod - def _message_has_cache_control(message: AllMessageValues) -> bool: + def _message_has_cache_control(message: AllMessageValues, openai_dialect: bool = False) -> bool: """Return True if the message already carries any cache_control.""" - return AnthropicCacheControlHook._count_cache_control_blocks(message) > 0 + return AnthropicCacheControlHook._count_cache_control_blocks(message, openai_dialect) > 0 @staticmethod def _safe_insert_cache_control_in_message( diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index c7820b8a447..ee274086ac8 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -230,14 +230,24 @@ def _translated(messages: Sequence[object], tools: Sequence[object]) -> frozense ) -def _wire_position(path: _Path, messages: Sequence[object]) -> tuple[int, ...]: - """Where Claude reads a mark: tool definitions, then system messages wherever they sit, then the other messages, - with a message-level mark on the message's last block.""" +def _leading_system_count(messages: Sequence[object]) -> int: + return next( + ( + index + for index, message in enumerate(messages) + if (_fields(message) or {}).get("role") not in ("system", "developer") + ), + len(messages), + ) + + +def _wire_position(path: _Path, leading_system_count: int) -> tuple[int, ...]: + """Where Claude reads a mark: tool definitions, then the leading system messages it hoists, then every other + message in place, with a message-level mark on the message's last block.""" if path[0] == "tools": return (0, *(index for index in path if isinstance(index, int))) index: Final = int(path[1]) - role: Final = (_fields(messages[index]) or {}).get("role") - group: Final = 1 if role in ("system", "developer") else 2 + group: Final = 1 if index < leading_system_count else 2 block: Final = ( (_MESSAGE_BLOCK_KEYS.index(str(path[2])), int(path[3])) if len(path) == 4 else (len(_MESSAGE_BLOCK_KEYS),) ) @@ -245,12 +255,12 @@ def _wire_position(path: _Path, messages: Sequence[object]) -> tuple[int, ...]: def _on_wire(path: _Path, messages: Sequence[object], tools: Sequence[object]) -> bool: - """Claude sends a message-level mark on list content only for a tool result, and a mark under ``function`` only - when the tool carries none of its own.""" + """Claude sends a message-level mark on string content, or on list content only for a tool result, and a mark + under ``function`` only when the tool carries none of its own.""" if path[0] == "tools": return len(path) == 2 or (_fields(tools[int(path[1])]) or {}).get("cache_control") is None message: Final = _fields(messages[int(path[1])]) or {} - return len(path) == 4 or not isinstance(message.get("content"), list) or message.get("role") == "tool" + return len(path) == 4 or isinstance(message.get("content"), str) or message.get("role") == "tool" def _ttl(cache_control: object) -> Literal["5m", "1h"]: @@ -316,13 +326,14 @@ def claude_prompt_cache( verbose_logger.debug("Claude: adjusting %s=%r because %s", problem.param, problem.value, problem.reason) renamed_messages, renamed_tools = _marked_request(messages, listed_tools, _renamed) translated: Final = _translated(messages, listed_tools) + leading_system_count: Final = _leading_system_count(messages) block_marks: Final = sorted( ( _Placed(path, _ttl(block.get("cache_control")), _kind(path, block.get("cache_control"), translated)) for path, block in _markable(renamed_messages, renamed_tools) if block.get("cache_control") is not None and _on_wire(path, renamed_messages, renamed_tools) ), - key=lambda mark: _wire_position(mark.path, messages), + key=lambda mark: _wire_position(mark.path, leading_system_count), ) request_mark: Final = params.get("cache_control") marks: Final = ( diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index bfac8f36433..cc4385e742f 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3310,6 +3310,33 @@ class TestOpenAIPromptCacheBreakpoint: assert result[3]["content"] == [{"type": "text", "text": "d", "prompt_cache_breakpoint": self.EXPLICIT}] assert result[4] == {"role": "user", "content": "e"} + def test_a_tool_call_breakpoint_takes_no_slot_openai_counts(self): + messages = [ + *( + {"role": "user", "content": [{"type": "text", "text": text, "prompt_cache_breakpoint": self.EXPLICIT}]} + for text in ("a", "b", "c") + ), + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_0", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + "prompt_cache_breakpoint": self.EXPLICIT, + } + ], + }, + {"role": "tool", "tool_call_id": "call_0", "content": "found"}, + {"role": "user", "content": "d"}, + ] + params = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + + _, processed, _ = self._chat(messages, params) + + assert processed[-1]["content"] == [{"type": "text", "text": "d", "prompt_cache_breakpoint": self.EXPLICIT}] + class TestOpenAIPromptCacheBreakpointPlacementRules: """OpenAI dialect only marks blocks OpenAI (and the /v1/messages bridges) can carry (#37509).""" diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index 75c6f47df69..fd1625a04dc 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -163,16 +163,43 @@ def test_a_breakpoint_read_before_a_one_hour_mark_is_sent_as_one_hour( later: dict[str, object], params: dict[str, object] ): messages: Final = [ - {"role": "user", "content": [_part("doc", prompt_cache_breakpoint=_EXPLICIT), _part("ask", **later)]}, {"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]}, + {"role": "user", "content": [_part("doc", prompt_cache_breakpoint=_EXPLICIT), _part("ask", **later)]}, ] result: Final = _resolved(messages, params=params) ask: Final = _part("ask", cache_control=_ONE_HOUR) if later else _part("ask") assert list(result.messages) == [ - {"role": "user", "content": [_part("doc", cache_control=_ONE_HOUR), ask]}, {"role": "system", "content": [_part("s", cache_control=_ONE_HOUR)]}, + {"role": "user", "content": [_part("doc", cache_control=_ONE_HOUR), ask]}, + ] + + +def test_a_breakpoint_read_before_a_later_system_messages_one_hour_mark_is_sent_as_one_hour(): + messages: Final = [ + {"role": "system", "content": [_part("s")]}, + {"role": "user", "content": [_part("doc", prompt_cache_breakpoint=_EXPLICIT)]}, + {"role": "system", "content": [_part("reminder", cache_control=_ONE_HOUR)]}, + {"role": "user", "content": [_part("ask")]}, + ] + + result: Final = _resolved(messages) + + assert list(result.messages)[1] == {"role": "user", "content": [_part("doc", cache_control=_ONE_HOUR)]} + + +def test_drop_params_drops_a_breakpoint_read_before_a_later_system_message_first(): + messages: Final = [ + {"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(4)]}, + {"role": "system", "content": [_part("reminder", prompt_cache_breakpoint=_EXPLICIT)]}, + ] + + result: Final = _resolved(messages, drop_params=True) + + assert list(result.messages) == [ + {"role": "user", "content": [_part("0"), *(_part(str(index), cache_control=_EPHEMERAL) for index in (1, 2, 3))]}, + {"role": "system", "content": [_part("reminder", cache_control=_EPHEMERAL)]}, ] @@ -291,6 +318,23 @@ def test_a_tool_marked_at_both_levels_takes_one_slot_under_drop_params(): ] +def test_a_message_level_breakpoint_on_a_tool_call_turn_without_content_takes_no_slot_under_drop_params(): + messages: Final = [ + {"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]}, + {"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(3)]}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "t", "arguments": "{}"}}], + "prompt_cache_breakpoint": _EXPLICIT, + }, + ] + + result: Final = _resolved(messages, drop_params=True) + + assert list(result.messages)[0] == {"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]} + + def test_a_null_tool_level_breakpoint_leaves_the_function_breakpoint_in_force(): tools: Final = [ {