mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(anthropic): preserve message cache control for list content
This commit is contained in:
parent
f4308bc124
commit
a62b79b90c
4 changed files with 125 additions and 0 deletions
|
|
@ -2407,6 +2407,7 @@ def anthropic_messages_pt(
|
|||
) = messages[msg_i]
|
||||
if user_message_types_block["role"] == "user":
|
||||
if isinstance(user_message_types_block["content"], list):
|
||||
user_content_start: Final = len(user_content)
|
||||
for m in user_message_types_block["content"]:
|
||||
if m.get("type", "") == "image_url":
|
||||
m = cast(ChatCompletionImageObject, m)
|
||||
|
|
@ -2478,6 +2479,16 @@ def anthropic_messages_pt(
|
|||
_file_content_element,
|
||||
)
|
||||
)
|
||||
if (
|
||||
len(user_content) > user_content_start
|
||||
and user_message_types_block.get("cache_control") is not None
|
||||
and not any(
|
||||
m.get("cache_control") is not None
|
||||
for m in user_message_types_block["content"]
|
||||
if isinstance(m, dict)
|
||||
)
|
||||
):
|
||||
add_cache_control_to_content(user_content[-1], dict(user_message_types_block))
|
||||
elif isinstance(user_message_types_block["content"], str):
|
||||
_anthropic_content_text_element: AnthropicMessagesTextParam = {
|
||||
"type": "text",
|
||||
|
|
@ -2680,6 +2691,7 @@ def anthropic_messages_pt(
|
|||
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
|
||||
assistant_content.extend(thinking_blocks)
|
||||
if _content_is_list and _content_list is not None:
|
||||
assistant_content_start: Final = len(assistant_content)
|
||||
for m in _content_list:
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
|
|
@ -2710,6 +2722,12 @@ def anthropic_messages_pt(
|
|||
# Pass through as-is since these are Anthropic-native content types
|
||||
elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"):
|
||||
assistant_content.append(m)
|
||||
if (
|
||||
len(assistant_content) > assistant_content_start
|
||||
and assistant_content_block.get("cache_control") is not None
|
||||
and not any(m.get("cache_control") is not None for m in _content_list if isinstance(m, dict))
|
||||
):
|
||||
add_cache_control_to_content(assistant_content[-1], dict(assistant_content_block))
|
||||
elif (
|
||||
"content" in assistant_content_block
|
||||
and isinstance(assistant_content_block["content"], str)
|
||||
|
|
@ -4417,6 +4435,16 @@ class BedrockConverseMessagesProcessor:
|
|||
)
|
||||
if _cache_point_block is not None:
|
||||
_parts.append(_cache_point_block)
|
||||
if (
|
||||
_parts
|
||||
and message_block.get("cache_control") is not None
|
||||
and not any("cachePoint" in part for part in _parts)
|
||||
):
|
||||
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
|
||||
message_block, block_type="content_block", model=model
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
_parts.append(_cache_point_block)
|
||||
user_content.extend(_parts)
|
||||
elif message_block["content"] and isinstance(message_block["content"], str):
|
||||
_part = BedrockContentBlock(text=messages[msg_i]["content"])
|
||||
|
|
@ -4790,6 +4818,16 @@ def _bedrock_converse_messages_pt(
|
|||
)
|
||||
if _cache_point_block is not None:
|
||||
_parts.append(_cache_point_block)
|
||||
if (
|
||||
_parts
|
||||
and message_block.get("cache_control") is not None
|
||||
and not any("cachePoint" in part for part in _parts)
|
||||
):
|
||||
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
|
||||
message_block, block_type="content_block", model=model
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
_parts.append(_cache_point_block)
|
||||
user_content.extend(_parts)
|
||||
elif message_block["content"] and isinstance(message_block["content"], str):
|
||||
_part = BedrockContentBlock(text=messages[msg_i]["content"])
|
||||
|
|
|
|||
|
|
@ -1719,6 +1719,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_system_message_content["cache_control"] = system_message_block["cache_control"]
|
||||
anthropic_system_message_list.append(anthropic_system_message_content)
|
||||
elif isinstance(message["content"], list):
|
||||
content_start: Final = len(anthropic_system_message_list)
|
||||
for _content in message["content"]:
|
||||
# Skip empty text blocks - Anthropic API raises errors for empty text
|
||||
text_value = _content.get("text")
|
||||
|
|
@ -1739,6 +1740,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_system_message_content["cache_control"] = _content["cache_control"]
|
||||
|
||||
anthropic_system_message_list.append(anthropic_system_message_content)
|
||||
if (
|
||||
len(anthropic_system_message_list) > content_start
|
||||
and system_message_block.get("cache_control") is not None
|
||||
and not any(
|
||||
content.get("cache_control") is not None
|
||||
for content in message["content"]
|
||||
if isinstance(content, dict)
|
||||
)
|
||||
):
|
||||
anthropic_system_message_list[-1]["cache_control"] = system_message_block["cache_control"]
|
||||
|
||||
if len(system_prompt_indices) > 0:
|
||||
for idx in reversed(system_prompt_indices):
|
||||
|
|
|
|||
|
|
@ -30,6 +30,56 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
from litellm.types.llms.openai import ChatCompletionToolMessage
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["user", "assistant"])
|
||||
@pytest.mark.parametrize("block_cache_control", [False, True])
|
||||
def test_anthropic_list_content_preserves_message_cache_control(role, block_cache_control):
|
||||
message_cache_control: Final = {"type": "ephemeral"}
|
||||
block_cache: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
content: Final = [
|
||||
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
|
||||
{"type": "text", "text": "second"},
|
||||
]
|
||||
message: Final = {"role": role, "content": content, "cache_control": message_cache_control}
|
||||
messages: Final = [message] if role == "user" else [{"role": "user", "content": "hi"}, message]
|
||||
|
||||
result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic")
|
||||
|
||||
expected: Final = [
|
||||
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "second",
|
||||
**({} if block_cache_control else {"cache_control": message_cache_control}),
|
||||
},
|
||||
]
|
||||
assert result[-1]["content"] == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("block_cache_control", [False, True])
|
||||
async def test_bedrock_list_content_preserves_message_cache_control(block_cache_control):
|
||||
content: Final = [
|
||||
{"type": "text", "text": "first", **({"cache_control": {"type": "ephemeral"}} if block_cache_control else {})},
|
||||
{"type": "text", "text": "second"},
|
||||
]
|
||||
messages: Final = [{"role": "user", "content": content, "cache_control": {"type": "ephemeral"}}]
|
||||
model: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
|
||||
sync_result: Final = _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock")
|
||||
async_result: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
|
||||
messages=messages, model=model, llm_provider="bedrock"
|
||||
)
|
||||
|
||||
cache_point: Final = {"cachePoint": {"type": "default"}}
|
||||
expected: Final = (
|
||||
[{"text": "first"}, cache_point, {"text": "second"}]
|
||||
if block_cache_control
|
||||
else [{"text": "first"}, {"text": "second"}, cache_point]
|
||||
)
|
||||
assert sync_result[0]["content"] == expected
|
||||
assert async_result[0]["content"] == expected
|
||||
|
||||
|
||||
def _get_gemini_function_response_inline_data_parts(result):
|
||||
assert isinstance(result, list), "expected Gemini parts list"
|
||||
assert len(result) == 1, "multimodal function responses should stay in one part"
|
||||
|
|
|
|||
|
|
@ -2383,6 +2383,32 @@ def test_translate_system_message_preserves_cache_control():
|
|||
assert result[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_cache_control", [False, True])
|
||||
def test_translate_system_list_content_preserves_message_cache_control(block_cache_control):
|
||||
message_cache_control: Final = {"type": "ephemeral"}
|
||||
block_cache: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
content: Final = [
|
||||
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
|
||||
{"type": "text", "text": "second"},
|
||||
]
|
||||
messages: Final = [
|
||||
{"role": "system", "content": content, "cache_control": message_cache_control},
|
||||
{"role": "user", "content": "hi"},
|
||||
]
|
||||
|
||||
result: Final = AnthropicConfig().translate_system_message(messages)
|
||||
|
||||
expected: Final = [
|
||||
{"type": "text", "text": "first", **({"cache_control": block_cache} if block_cache_control else {})},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "second",
|
||||
**({} if block_cache_control else {"cache_control": message_cache_control}),
|
||||
},
|
||||
]
|
||||
assert result == expected
|
||||
|
||||
|
||||
# ============ Dynamic max_tokens Tests ============
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue