From c92bbe00e24bfb5a584fd675ec79c7b2eb970f01 Mon Sep 17 00:00:00 2001 From: Oxygenesis Date: Sun, 27 Sep 2026 23:08:52 +0700 Subject: [PATCH 1/2] fix(anthropic): preserve message cache control for list content --- .../prompt_templates/factory.py | 64 +++++++++++++++++++ litellm/llms/anthropic/chat/transformation.py | 12 ++++ ...llm_core_utils_prompt_templates_factory.py | 32 ++++++++++ .../test_anthropic_chat_transformation.py | 24 +++++++ .../chat/test_converse_transformation.py | 34 ++++++++++ 5 files changed, 166 insertions(+) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c4e242fd360..b499c654493 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1860,6 +1860,30 @@ def add_cache_control_to_content( return anthropic_content_element +def _apply_message_cache_control_to_last_content_block( + converted_content: list[dict], + message: dict, + original_content: list, + start_index: int, +) -> None: + """Apply message-level cache control when list content has no block-level value.""" + if message.get("cache_control") is None: + return + if any( + isinstance(content_element, dict) and content_element.get("cache_control") is not None + for content_element in original_content + ): + return + + for content_element in reversed(converted_content[start_index:]): + if isinstance(content_element, dict): + add_cache_control_to_content( + anthropic_content_element=content_element, + original_content_element=message, + ) + return + + def _anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: @@ -2407,6 +2431,7 @@ def anthropic_messages_pt( ) = messages[msg_i] if user_message_types_block["role"] == "user": if isinstance(user_message_types_block["content"], list): + message_content_start = len(user_content) for m in user_message_types_block["content"]: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) @@ -2478,6 +2503,12 @@ def anthropic_messages_pt( _file_content_element, ) ) + _apply_message_cache_control_to_last_content_block( + converted_content=user_content, + message=user_message_types_block, + original_content=user_message_types_block["content"], + start_index=message_content_start, + ) elif isinstance(user_message_types_block["content"], str): _anthropic_content_text_element: AnthropicMessagesTextParam = { "type": "text", @@ -2680,6 +2711,7 @@ def anthropic_messages_pt( ): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR assistant_content.extend(thinking_blocks) if _content_is_list and _content_list is not None: + message_content_start = len(assistant_content) for m in _content_list: if not isinstance(m, dict): continue @@ -2710,6 +2742,12 @@ def anthropic_messages_pt( # Pass through as-is since these are Anthropic-native content types elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"): assistant_content.append(m) + _apply_message_cache_control_to_last_content_block( + converted_content=assistant_content, + message=assistant_content_block, + original_content=_content_list, + start_index=message_content_start, + ) elif ( "content" in assistant_content_block and isinstance(assistant_content_block["content"], str) @@ -4417,6 +4455,19 @@ class BedrockConverseMessagesProcessor: ) if _cache_point_block is not None: _parts.append(_cache_point_block) + message_cache_control = message_block.get("cache_control") + has_content_cache_control = any( + isinstance(element, dict) and element.get("cache_control") is not None + for element in message_block["content"] + ) + if isinstance(message_cache_control, dict) and not has_content_cache_control: + _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( + {"cache_control": message_cache_control}, + block_type="content_block", + model=model, + ) + if _cache_point_block is not None: + _parts.append(_cache_point_block) user_content.extend(_parts) elif message_block["content"] and isinstance(message_block["content"], str): _part = BedrockContentBlock(text=messages[msg_i]["content"]) @@ -4790,6 +4841,19 @@ def _bedrock_converse_messages_pt( ) if _cache_point_block is not None: _parts.append(_cache_point_block) + message_cache_control = message_block.get("cache_control") + has_content_cache_control = any( + isinstance(element, dict) and element.get("cache_control") is not None + for element in message_block["content"] + ) + if isinstance(message_cache_control, dict) and not has_content_cache_control: + _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( + {"cache_control": message_cache_control}, + block_type="content_block", + model=model, + ) + if _cache_point_block is not None: + _parts.append(_cache_point_block) user_content.extend(_parts) elif message_block["content"] and isinstance(message_block["content"], str): _part = BedrockContentBlock(text=messages[msg_i]["content"]) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 3bffee48d6a..166acfae9bb 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1719,6 +1719,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_system_message_content["cache_control"] = system_message_block["cache_control"] anthropic_system_message_list.append(anthropic_system_message_content) elif isinstance(message["content"], list): + message_content_start = len(anthropic_system_message_list) + has_content_cache_control = any( + isinstance(content, dict) and content.get("cache_control") is not None + for content in message["content"] + ) for _content in message["content"]: # Skip empty text blocks - Anthropic API raises errors for empty text text_value = _content.get("text") @@ -1740,6 +1745,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_system_message_list.append(anthropic_system_message_content) + message_cache_control = system_message_block.get("cache_control") + if isinstance(message_cache_control, dict) and not has_content_cache_control: + for content in reversed(anthropic_system_message_list[message_content_start:]): + if isinstance(content, dict): + content["cache_control"] = message_cache_control + break + if len(system_prompt_indices) > 0: for idx in reversed(system_prompt_indices): messages.pop(idx) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 1e12a973cdb..ed89d7d4aef 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -228,6 +228,38 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): ) +def test_anthropic_messages_pt_applies_message_cache_control_to_last_list_block(): + """Message-level cache control should survive list-content translation.""" + cache_control = {"type": "ephemeral"} + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "first user block"}, + {"type": "text", "text": "last user block"}, + ], + "cache_control": cache_control, + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "first assistant block"}, + {"type": "text", "text": "last assistant block"}, + ], + "cache_control": cache_control, + }, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + user_content = result[0]["content"] + assistant_content = result[1]["content"] + assert "cache_control" not in user_content[0] + assert user_content[1]["cache_control"] == cache_control + assert "cache_control" not in assistant_content[0] + assert assistant_content[1]["cache_control"] == cache_control + + def test_anthropic_messages_pt_keeps_signed_thinking_block(): """A genuine Anthropic round-trip still holds its original signature, so that thinking block must be forwarded unchanged (we only drop unsignable blocks). diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 332153b4c7d..79c3f38e860 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2383,6 +2383,30 @@ def test_translate_system_message_preserves_cache_control(): assert result[0]["cache_control"] == {"type": "ephemeral"} +def test_translate_system_message_applies_message_cache_control_to_last_list_block(): + """Message-level cache control should apply to the last system content block.""" + config = AnthropicConfig() + cache_control = {"type": "ephemeral"} + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "First block"}, + {"type": "text", "text": "Last block"}, + ], + "cache_control": cache_control, + }, + {"role": "user", "content": "Hello"}, + ] + + result = config.translate_system_message(messages) + + assert result[0]["text"] == "First block" + assert "cache_control" not in result[0] + assert result[1]["text"] == "Last block" + assert result[1]["cache_control"] == cache_control + + # ============ Dynamic max_tokens Tests ============ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 499096621c5..5321c57af5f 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -2376,6 +2376,40 @@ async def test_assistant_message_list_content_cache_control(): assert assistant_content[1]["cachePoint"]["type"] == "default" +@pytest.mark.asyncio +async def test_user_message_list_content_cache_control(): + """Message-level cache control should survive Bedrock list-content translation.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + _bedrock_converse_messages_pt, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "First block"}, + {"type": "text", "text": "Last block"}, + ], + "cache_control": {"type": "ephemeral"}, + } + ] + model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + result = _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock_converse") + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model=model, + llm_provider="bedrock_converse", + ) + + assert result == async_result + content = result[0]["content"] + assert content[0]["text"] == "First block" + assert content[1]["text"] == "Last block" + assert content[2]["cachePoint"]["type"] == "default" + + @pytest.mark.asyncio async def test_tool_message_cache_control(): """Test that tool messages with cache_control generate cachePoint blocks.""" From ed552550d2f7deb7c8dd0a71993053a5cb493627 Mon Sep 17 00:00:00 2001 From: Oxygenesis Date: Tue, 29 Sep 2026 23:35:23 +0700 Subject: [PATCH 2/2] fix(bedrock): reuse typed message cache block --- litellm/litellm_core_utils/prompt_templates/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index b499c654493..729bdda406e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4462,7 +4462,7 @@ class BedrockConverseMessagesProcessor: ) if isinstance(message_cache_control, dict) and not has_content_cache_control: _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - {"cache_control": message_cache_control}, + message_block, block_type="content_block", model=model, ) @@ -4848,7 +4848,7 @@ def _bedrock_converse_messages_pt( ) if isinstance(message_cache_control, dict) and not has_content_cache_control: _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - {"cache_control": message_cache_control}, + message_block, block_type="content_block", model=model, )