From a62b79b90c0647dbbe250ba0fa5fc33379d9a8f3 Mon Sep 17 00:00:00 2001 From: amazingniuma-art Date: Sun, 27 Sep 2026 17:31:25 +0800 Subject: [PATCH] fix(anthropic): preserve message cache control for list content --- .../prompt_templates/factory.py | 38 ++++++++++++++ litellm/llms/anthropic/chat/transformation.py | 11 ++++ ...llm_core_utils_prompt_templates_factory.py | 50 +++++++++++++++++++ .../test_anthropic_chat_transformation.py | 26 ++++++++++ 4 files changed, 125 insertions(+) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c4e242fd360..cf5580b6692 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2407,6 +2407,7 @@ def anthropic_messages_pt( ) = messages[msg_i] if user_message_types_block["role"] == "user": if isinstance(user_message_types_block["content"], list): + user_content_start: Final = len(user_content) for m in user_message_types_block["content"]: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) @@ -2478,6 +2479,16 @@ def anthropic_messages_pt( _file_content_element, ) ) + if ( + len(user_content) > user_content_start + and user_message_types_block.get("cache_control") is not None + and not any( + m.get("cache_control") is not None + for m in user_message_types_block["content"] + if isinstance(m, dict) + ) + ): + add_cache_control_to_content(user_content[-1], dict(user_message_types_block)) elif isinstance(user_message_types_block["content"], str): _anthropic_content_text_element: AnthropicMessagesTextParam = { "type": "text", @@ -2680,6 +2691,7 @@ def anthropic_messages_pt( ): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR assistant_content.extend(thinking_blocks) if _content_is_list and _content_list is not None: + assistant_content_start: Final = len(assistant_content) for m in _content_list: if not isinstance(m, dict): continue @@ -2710,6 +2722,12 @@ def anthropic_messages_pt( # Pass through as-is since these are Anthropic-native content types elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"): assistant_content.append(m) + if ( + len(assistant_content) > assistant_content_start + and assistant_content_block.get("cache_control") is not None + and not any(m.get("cache_control") is not None for m in _content_list if isinstance(m, dict)) + ): + add_cache_control_to_content(assistant_content[-1], dict(assistant_content_block)) elif ( "content" in assistant_content_block and isinstance(assistant_content_block["content"], str) @@ -4417,6 +4435,16 @@ class BedrockConverseMessagesProcessor: ) if _cache_point_block is not None: _parts.append(_cache_point_block) + if ( + _parts + and message_block.get("cache_control") is not None + and not any("cachePoint" in part for part in _parts) + ): + _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( + message_block, block_type="content_block", model=model + ) + if _cache_point_block is not None: + _parts.append(_cache_point_block) user_content.extend(_parts) elif message_block["content"] and isinstance(message_block["content"], str): _part = BedrockContentBlock(text=messages[msg_i]["content"]) @@ -4790,6 +4818,16 @@ def _bedrock_converse_messages_pt( ) if _cache_point_block is not None: _parts.append(_cache_point_block) + if ( + _parts + and message_block.get("cache_control") is not None + and not any("cachePoint" in part for part in _parts) + ): + _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( + message_block, block_type="content_block", model=model + ) + if _cache_point_block is not None: + _parts.append(_cache_point_block) user_content.extend(_parts) elif message_block["content"] and isinstance(message_block["content"], str): _part = BedrockContentBlock(text=messages[msg_i]["content"]) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 3bffee48d6a..24902c87ac2 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1719,6 +1719,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_system_message_content["cache_control"] = system_message_block["cache_control"] anthropic_system_message_list.append(anthropic_system_message_content) elif isinstance(message["content"], list): + content_start: Final = len(anthropic_system_message_list) for _content in message["content"]: # Skip empty text blocks - Anthropic API raises errors for empty text text_value = _content.get("text") @@ -1739,6 +1740,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_system_message_content["cache_control"] = _content["cache_control"] anthropic_system_message_list.append(anthropic_system_message_content) + if ( + len(anthropic_system_message_list) > content_start + and system_message_block.get("cache_control") is not None + and not any( + content.get("cache_control") is not None + for content in message["content"] + if isinstance(content, dict) + ) + ): + anthropic_system_message_list[-1]["cache_control"] = system_message_block["cache_control"] if len(system_prompt_indices) > 0: for idx in reversed(system_prompt_indices): diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 1e12a973cdb..937ed191c97 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -30,6 +30,56 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( from litellm.types.llms.openai import ChatCompletionToolMessage +@pytest.mark.parametrize("role", ["user", "assistant"]) +@pytest.mark.parametrize("block_cache_control", [False, True]) +def test_anthropic_list_content_preserves_message_cache_control(role, block_cache_control): + message_cache_control: Final = {"type": "ephemeral"} + block_cache: Final = {"type": "ephemeral", "ttl": "1h"} + content: Final = [ + {"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})}, + {"type": "text", "text": "second"}, + ] + message: Final = {"role": role, "content": content, "cache_control": message_cache_control} + messages: Final = [message] if role == "user" else [{"role": "user", "content": "hi"}, message] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + expected: Final = [ + {"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})}, + { + "type": "text", + "text": "second", + **({} if block_cache_control else {"cache_control": message_cache_control}), + }, + ] + assert result[-1]["content"] == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("block_cache_control", [False, True]) +async def test_bedrock_list_content_preserves_message_cache_control(block_cache_control): + content: Final = [ + {"type": "text", "text": "first", **({"cache_control": {"type": "ephemeral"}} if block_cache_control else {})}, + {"type": "text", "text": "second"}, + ] + messages: Final = [{"role": "user", "content": content, "cache_control": {"type": "ephemeral"}}] + model: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" + + sync_result: Final = _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + async_result: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + + cache_point: Final = {"cachePoint": {"type": "default"}} + expected: Final = ( + [{"text": "first"}, cache_point, {"text": "second"}] + if block_cache_control + else [{"text": "first"}, {"text": "second"}, cache_point] + ) + assert sync_result[0]["content"] == expected + assert async_result[0]["content"] == expected + + def _get_gemini_function_response_inline_data_parts(result): assert isinstance(result, list), "expected Gemini parts list" assert len(result) == 1, "multimodal function responses should stay in one part" diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 332153b4c7d..659721274fa 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2383,6 +2383,32 @@ def test_translate_system_message_preserves_cache_control(): assert result[0]["cache_control"] == {"type": "ephemeral"} +@pytest.mark.parametrize("block_cache_control", [False, True]) +def test_translate_system_list_content_preserves_message_cache_control(block_cache_control): + message_cache_control: Final = {"type": "ephemeral"} + block_cache: Final = {"type": "ephemeral", "ttl": "1h"} + content: Final = [ + {"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})}, + {"type": "text", "text": "second"}, + ] + messages: Final = [ + {"role": "system", "content": content, "cache_control": message_cache_control}, + {"role": "user", "content": "hi"}, + ] + + result: Final = AnthropicConfig().translate_system_message(messages) + + expected: Final = [ + {"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})}, + { + "type": "text", + "text": "second", + **({} if block_cache_control else {"cache_control": message_cache_control}), + }, + ] + assert result == expected + + # ============ Dynamic max_tokens Tests ============