From 7bb44abdcc078ea27eb79eeb34c97acab0c10c69 Mon Sep 17 00:00:00 2001 From: amazingniuma-art Date: Sun, 27 Sep 2026 18:38:10 +0800 Subject: [PATCH] fix(anthropic): address cache breakpoint review findings --- .../prompt_templates/factory.py | 76 ++++++++++--- litellm/llms/anthropic/chat/transformation.py | 3 +- ...llm_core_utils_prompt_templates_factory.py | 105 ++++++++++++++++++ .../test_anthropic_chat_transformation.py | 18 +++ 4 files changed, 185 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 725da3cd7a7..b3a300bbe46 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2483,12 +2483,18 @@ def anthropic_messages_pt( len(user_content) > user_content_start and user_message_types_block.get("cache_control") is not None and not any( - m.get("cache_control") is not None - for m in user_message_types_block["content"] - if isinstance(m, dict) + "cache_control" in content and content["cache_control"] is not None + for content in user_content[user_content_start:] ) ): - add_cache_control_to_content(user_content[-1], dict(user_message_types_block)) + cached_user_content: AnthropicMessagesUserMessageValues = copy.copy(user_content[-1]) + add_cache_control_to_content( + anthropic_content_element=cast( + dict[str, object], cached_user_content + ), # cast-ok: copied TypedDict is a dict + original_content_element=user_message_types_block, + ) + user_content[-1] = cached_user_content elif isinstance(user_message_types_block["content"], str): _anthropic_content_text_element: AnthropicMessagesTextParam = { "type": "text", @@ -2549,6 +2555,12 @@ def anthropic_messages_pt( _has_server_tool_calls = True break + _content_is_list: bool = "content" in assistant_content_block and isinstance( + assistant_content_block["content"], list + ) + _content_list = assistant_content_block.get("content") if _content_is_list else None + assistant_content_start: int = len(assistant_content) + if ( thinking_blocks is not None and _has_server_tool_calls @@ -2672,10 +2684,6 @@ def anthropic_messages_pt( # duplication and preserve the original interleaved order. # Fixes the gap where list-content messages bypass INTERLEAVED # MODE and still get thinking blocks prepended out of order. - _content_is_list = "content" in assistant_content_block and isinstance( - assistant_content_block["content"], list - ) - _content_list = assistant_content_block.get("content") if _content_is_list else None _list_has_thinking = False if _content_is_list and _content_list is not None: for _item in _content_list: @@ -2691,7 +2699,7 @@ def anthropic_messages_pt( ): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR assistant_content.extend(thinking_blocks) if _content_is_list and _content_list is not None: - assistant_content_start: int = len(assistant_content) + assistant_content_start = len(assistant_content) for m in _content_list: if not isinstance(m, dict): continue @@ -2722,12 +2730,6 @@ def anthropic_messages_pt( # Pass through as-is since these are Anthropic-native content types elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"): assistant_content.append(m) - if ( - len(assistant_content) > assistant_content_start - and assistant_content_block.get("cache_control") is not None - and not any(m.get("cache_control") is not None for m in _content_list if isinstance(m, dict)) - ): - add_cache_control_to_content(assistant_content[-1], dict(assistant_content_block)) elif ( "content" in assistant_content_block and isinstance(assistant_content_block["content"], str) @@ -2782,6 +2784,24 @@ def anthropic_messages_pt( if assistant_function_call is not None: assistant_content.extend(convert_function_to_anthropic_tool_invoke(assistant_function_call)) + if ( + _content_is_list + and len(assistant_content) > assistant_content_start + and assistant_content_block.get("cache_control") is not None + and not any( + "cache_control" in content and content["cache_control"] is not None + for content in assistant_content[assistant_content_start:] + ) + ): + cached_assistant_content: AnthropicMessagesAssistantMessageValues = copy.copy(assistant_content[-1]) + add_cache_control_to_content( + anthropic_content_element=cast( + dict[str, object], cached_assistant_content + ), # cast-ok: copied TypedDict is a dict + original_content_element=assistant_content_block, + ) + assistant_content[-1] = cached_assistant_content + msg_i += 1 if assistant_content: @@ -4530,6 +4550,7 @@ class BedrockConverseMessagesProcessor: message=messages[msg_i], assistant_continue_message=assistant_continue_message, ) + assistant_message_content_start: int = len(assistant_content) _assistant_content = assistant_message_block.get("content", None) thinking_blocks = cast( list[ChatCompletionThinkingBlock] | None, @@ -4598,6 +4619,18 @@ class BedrockConverseMessagesProcessor: if _tool_calls: assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model)) + if ( + isinstance(_assistant_content, list) + and len(assistant_content) > assistant_message_content_start + and assistant_message_block.get("cache_control") is not None + and not any("cachePoint" in block for block in assistant_content[assistant_message_content_start:]) + ): + _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( + assistant_message_block, block_type="content_block", model=model + ) + if _cache_point_block is not None: + assistant_content.append(_cache_point_block) + msg_i += 1 assistant_content = _deduplicate_bedrock_content_blocks(assistant_content, "toolUse") @@ -4915,6 +4948,7 @@ def _bedrock_converse_messages_pt( message=messages[msg_i], assistant_continue_message=assistant_continue_message, ) + assistant_message_content_start: int = len(assistant_content) _assistant_content = assistant_message_block.get("content", None) thinking_blocks = cast( list[ChatCompletionThinkingBlock] | None, @@ -4984,6 +5018,18 @@ def _bedrock_converse_messages_pt( if _tool_calls: assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model)) + if ( + isinstance(_assistant_content, list) + and len(assistant_content) > assistant_message_content_start + and assistant_message_block.get("cache_control") is not None + and not any("cachePoint" in block for block in assistant_content[assistant_message_content_start:]) + ): + _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( + assistant_message_block, block_type="content_block", model=model + ) + if _cache_point_block is not None: + assistant_content.append(_cache_point_block) + msg_i += 1 assistant_content = _deduplicate_bedrock_content_blocks(assistant_content, "toolUse") diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 8ec20eff068..a83f9ec5a54 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1745,8 +1745,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and system_message_block.get("cache_control") is not None and not any( content.get("cache_control") is not None - for content in message["content"] - if isinstance(content, dict) + for content in anthropic_system_message_list[content_start:] ) ): anthropic_system_message_list[-1]["cache_control"] = system_message_block["cache_control"] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 937ed191c97..b310080687b 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -55,6 +55,73 @@ def test_anthropic_list_content_preserves_message_cache_control(role, block_cach assert result[-1]["content"] == expected +@pytest.mark.parametrize("call_field", ["tool_calls", "function_call"]) +def test_anthropic_list_message_cache_control_follows_tool_calls(call_field): + cache_control: Final = {"type": "ephemeral"} + call: Final = ( + [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}] + if call_field == "tool_calls" + else {"name": "lookup", "arguments": "{}"} + ) + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [{"type": "text", "text": "checking"}], + "cache_control": cache_control, + call_field: call, + }, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + blocks: Final = result[-1]["content"] + + assert blocks[0] == {"type": "text", "text": "checking"} + assert blocks[-1]["type"] == "tool_use" + assert blocks[-1]["cache_control"] == cache_control + + +@pytest.mark.parametrize("role", ["user", "assistant"]) +def test_anthropic_list_message_cache_control_preserves_caller_block(role): + block: Final = ( + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "document"}} + if role == "user" + else {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}} + ) + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": role, "content": [block], "cache_control": {"type": "ephemeral"}}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert "cache_control" not in block + assert result[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert result[-1]["content"][-1] is not block + + +@pytest.mark.parametrize("role", ["user", "assistant"]) +def test_anthropic_skipped_list_block_does_not_suppress_message_cache_control(role): + skipped_block: Final = ( + {"type": "unknown", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + if role == "user" + else {"type": "thinking", "thinking": "unsigned", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ) + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": role, + "content": [skipped_block, {"type": "text", "text": "retained"}], + "cache_control": {"type": "ephemeral"}, + }, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert result[-1]["content"][-1] == {"type": "text", "text": "retained", "cache_control": {"type": "ephemeral"}} + assert all(block.get("type") != skipped_block["type"] for block in result[-1]["content"]) + + @pytest.mark.asyncio @pytest.mark.parametrize("block_cache_control", [False, True]) async def test_bedrock_list_content_preserves_message_cache_control(block_cache_control): @@ -80,6 +147,44 @@ async def test_bedrock_list_content_preserves_message_cache_control(block_cache_ assert async_result[0]["content"] == expected +@pytest.mark.asyncio +@pytest.mark.parametrize("block_cache_control", [False, True]) +@pytest.mark.parametrize("with_tool_calls", [False, True]) +async def test_bedrock_assistant_list_preserves_message_cache_control(block_cache_control, with_tool_calls): + content: Final = [ + { + "type": "text", + "text": "checking", + **({"cache_control": {"type": "ephemeral"}} if block_cache_control else {}), + } + ] + assistant_message: Final = { + "role": "assistant", + "content": content, + "cache_control": {"type": "ephemeral"}, + **( + {"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}]} + if with_tool_calls + else {} + ), + } + messages: Final = [{"role": "user", "content": "hi"}, assistant_message] + model: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" + + sync_result: Final = _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + async_result: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + + for result in (sync_result, async_result): + blocks = result[-1]["content"] + assert sum("cachePoint" in block for block in blocks) == 1 + if not block_cache_control: + assert blocks[-1] == {"cachePoint": {"type": "default"}} + if with_tool_calls: + assert "toolUse" in blocks[-2] + + def _get_gemini_function_response_inline_data_parts(result): assert isinstance(result, list), "expected Gemini parts list" assert len(result) == 1, "multimodal function responses should stay in one part" diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 659721274fa..7fdf8032dba 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2409,6 +2409,24 @@ def test_translate_system_list_content_preserves_message_cache_control(block_cac assert result == expected +def test_translate_system_skipped_list_block_does_not_suppress_message_cache_control(): + messages: Final = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "retained"}, + ], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "user", "content": "hi"}, + ] + + result: Final = AnthropicConfig().translate_system_message(messages) + + assert result == [{"type": "text", "text": "retained", "cache_control": {"type": "ephemeral"}}] + + # ============ Dynamic max_tokens Tests ============