fix(anthropic): preserve message cache control for list content

This commit is contained in:
amazingniuma-art 2026-09-27 17:31:25 +08:00
parent f4308bc124
commit a62b79b90c
4 changed files with 125 additions and 0 deletions

View file

@ -2407,6 +2407,7 @@ def anthropic_messages_pt(
) = messages[msg_i]
if user_message_types_block["role"] == "user":
if isinstance(user_message_types_block["content"], list):
user_content_start: Final = len(user_content)
for m in user_message_types_block["content"]:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
@ -2478,6 +2479,16 @@ def anthropic_messages_pt(
_file_content_element,
)
)
if (
len(user_content) > user_content_start
and user_message_types_block.get("cache_control") is not None
and not any(
m.get("cache_control") is not None
for m in user_message_types_block["content"]
if isinstance(m, dict)
)
):
add_cache_control_to_content(user_content[-1], dict(user_message_types_block))
elif isinstance(user_message_types_block["content"], str):
_anthropic_content_text_element: AnthropicMessagesTextParam = {
"type": "text",
@ -2680,6 +2691,7 @@ def anthropic_messages_pt(
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(thinking_blocks)
if _content_is_list and _content_list is not None:
assistant_content_start: Final = len(assistant_content)
for m in _content_list:
if not isinstance(m, dict):
continue
@ -2710,6 +2722,12 @@ def anthropic_messages_pt(
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"):
assistant_content.append(m)
if (
len(assistant_content) > assistant_content_start
and assistant_content_block.get("cache_control") is not None
and not any(m.get("cache_control") is not None for m in _content_list if isinstance(m, dict))
):
add_cache_control_to_content(assistant_content[-1], dict(assistant_content_block))
elif (
"content" in assistant_content_block
and isinstance(assistant_content_block["content"], str)
@ -4417,6 +4435,16 @@ class BedrockConverseMessagesProcessor:
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
if (
_parts
and message_block.get("cache_control") is not None
and not any("cachePoint" in part for part in _parts)
):
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
message_block, block_type="content_block", model=model
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
user_content.extend(_parts)
elif message_block["content"] and isinstance(message_block["content"], str):
_part = BedrockContentBlock(text=messages[msg_i]["content"])
@ -4790,6 +4818,16 @@ def _bedrock_converse_messages_pt(
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
if (
_parts
and message_block.get("cache_control") is not None
and not any("cachePoint" in part for part in _parts)
):
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
message_block, block_type="content_block", model=model
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
user_content.extend(_parts)
elif message_block["content"] and isinstance(message_block["content"], str):
_part = BedrockContentBlock(text=messages[msg_i]["content"])

View file

@ -1719,6 +1719,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_system_message_content["cache_control"] = system_message_block["cache_control"]
anthropic_system_message_list.append(anthropic_system_message_content)
elif isinstance(message["content"], list):
content_start: Final = len(anthropic_system_message_list)
for _content in message["content"]:
# Skip empty text blocks - Anthropic API raises errors for empty text
text_value = _content.get("text")
@ -1739,6 +1740,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_system_message_content["cache_control"] = _content["cache_control"]
anthropic_system_message_list.append(anthropic_system_message_content)
if (
len(anthropic_system_message_list) > content_start
and system_message_block.get("cache_control") is not None
and not any(
content.get("cache_control") is not None
for content in message["content"]
if isinstance(content, dict)
)
):
anthropic_system_message_list[-1]["cache_control"] = system_message_block["cache_control"]
if len(system_prompt_indices) > 0:
for idx in reversed(system_prompt_indices):

View file

@ -30,6 +30,56 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
from litellm.types.llms.openai import ChatCompletionToolMessage
@pytest.mark.parametrize("role", ["user", "assistant"])
@pytest.mark.parametrize("block_cache_control", [False, True])
def test_anthropic_list_content_preserves_message_cache_control(role, block_cache_control):
message_cache_control: Final = {"type": "ephemeral"}
block_cache: Final = {"type": "ephemeral", "ttl": "1h"}
content: Final = [
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
{"type": "text", "text": "second"},
]
message: Final = {"role": role, "content": content, "cache_control": message_cache_control}
messages: Final = [message] if role == "user" else [{"role": "user", "content": "hi"}, message]
result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic")
expected: Final = [
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
{
"type": "text",
"text": "second",
**({} if block_cache_control else {"cache_control": message_cache_control}),
},
]
assert result[-1]["content"] == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("block_cache_control", [False, True])
async def test_bedrock_list_content_preserves_message_cache_control(block_cache_control):
content: Final = [
{"type": "text", "text": "first", **({"cache_control": {"type": "ephemeral"}} if block_cache_control else {})},
{"type": "text", "text": "second"},
]
messages: Final = [{"role": "user", "content": content, "cache_control": {"type": "ephemeral"}}]
model: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
sync_result: Final = _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock")
async_result: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
messages=messages, model=model, llm_provider="bedrock"
)
cache_point: Final = {"cachePoint": {"type": "default"}}
expected: Final = (
[{"text": "first"}, cache_point, {"text": "second"}]
if block_cache_control
else [{"text": "first"}, {"text": "second"}, cache_point]
)
assert sync_result[0]["content"] == expected
assert async_result[0]["content"] == expected
def _get_gemini_function_response_inline_data_parts(result):
assert isinstance(result, list), "expected Gemini parts list"
assert len(result) == 1, "multimodal function responses should stay in one part"

View file

@ -2383,6 +2383,32 @@ def test_translate_system_message_preserves_cache_control():
assert result[0]["cache_control"] == {"type": "ephemeral"}
@pytest.mark.parametrize("block_cache_control", [False, True])
def test_translate_system_list_content_preserves_message_cache_control(block_cache_control):
message_cache_control: Final = {"type": "ephemeral"}
block_cache: Final = {"type": "ephemeral", "ttl": "1h"}
content: Final = [
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
{"type": "text", "text": "second"},
]
messages: Final = [
{"role": "system", "content": content, "cache_control": message_cache_control},
{"role": "user", "content": "hi"},
]
result: Final = AnthropicConfig().translate_system_message(messages)
expected: Final = [
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
{
"type": "text",
"text": "second",
**({} if block_cache_control else {"cache_control": message_cache_control}),
},
]
assert result == expected
# ============ Dynamic max_tokens Tests ============