scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(anthropic): address cache breakpoint review findings

Browse source
This commit is contained in:
amazingniuma-art 2026-09-27 18:38:10 +08:00
parent cb8d5f42d8
commit 7bb44abdcc
4 changed files with 185 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2483,12 +2483,18 @@ def anthropic_messages_pt(
len(user_content) > user_content_start
and user_message_types_block.get("cache_control") is not None
and not any(
m.get("cache_control") is not None
for m in user_message_types_block["content"]
if isinstance(m, dict)
"cache_control" in content and content["cache_control"] is not None
for content in user_content[user_content_start:]
)
):
add_cache_control_to_content(user_content[-1], dict(user_message_types_block))
cached_user_content: AnthropicMessagesUserMessageValues = copy.copy(user_content[-1])
add_cache_control_to_content(
anthropic_content_element=cast(
dict[str, object], cached_user_content
), # cast-ok: copied TypedDict is a dict
original_content_element=user_message_types_block,
)
user_content[-1] = cached_user_content
elif isinstance(user_message_types_block["content"], str):
_anthropic_content_text_element: AnthropicMessagesTextParam = {
"type": "text",
@ -2549,6 +2555,12 @@ def anthropic_messages_pt(
_has_server_tool_calls = True
break
_content_is_list: bool = "content" in assistant_content_block and isinstance(
assistant_content_block["content"], list
)
_content_list = assistant_content_block.get("content") if _content_is_list else None
assistant_content_start: int = len(assistant_content)
if (
thinking_blocks is not None
and _has_server_tool_calls
@ -2672,10 +2684,6 @@ def anthropic_messages_pt(
# duplication and preserve the original interleaved order.
# Fixes the gap where list-content messages bypass INTERLEAVED
# MODE and still get thinking blocks prepended out of order.
_content_is_list = "content" in assistant_content_block and isinstance(
assistant_content_block["content"], list
)
_content_list = assistant_content_block.get("content") if _content_is_list else None
_list_has_thinking = False
if _content_is_list and _content_list is not None:
for _item in _content_list:
@ -2691,7 +2699,7 @@ def anthropic_messages_pt(
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(thinking_blocks)
if _content_is_list and _content_list is not None:
assistant_content_start: int = len(assistant_content)
assistant_content_start = len(assistant_content)
for m in _content_list:
if not isinstance(m, dict):
continue
@ -2722,12 +2730,6 @@ def anthropic_messages_pt(
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "server_tool_use" or m.get("type", "").endswith("_tool_result"):
assistant_content.append(m)
if (
len(assistant_content) > assistant_content_start
and assistant_content_block.get("cache_control") is not None
and not any(m.get("cache_control") is not None for m in _content_list if isinstance(m, dict))
):
add_cache_control_to_content(assistant_content[-1], dict(assistant_content_block))
elif (
"content" in assistant_content_block
and isinstance(assistant_content_block["content"], str)
@ -2782,6 +2784,24 @@ def anthropic_messages_pt(
if assistant_function_call is not None:
assistant_content.extend(convert_function_to_anthropic_tool_invoke(assistant_function_call))
if (
_content_is_list
and len(assistant_content) > assistant_content_start
and assistant_content_block.get("cache_control") is not None
and not any(
"cache_control" in content and content["cache_control"] is not None
for content in assistant_content[assistant_content_start:]
)
):
cached_assistant_content: AnthropicMessagesAssistantMessageValues = copy.copy(assistant_content[-1])
add_cache_control_to_content(
anthropic_content_element=cast(
dict[str, object], cached_assistant_content
), # cast-ok: copied TypedDict is a dict
original_content_element=assistant_content_block,
)
assistant_content[-1] = cached_assistant_content
msg_i += 1
if assistant_content:
@ -4530,6 +4550,7 @@ class BedrockConverseMessagesProcessor:
message=messages[msg_i],
assistant_continue_message=assistant_continue_message,
)
assistant_message_content_start: int = len(assistant_content)
_assistant_content = assistant_message_block.get("content", None)
thinking_blocks = cast(
list[ChatCompletionThinkingBlock] | None,
@ -4598,6 +4619,18 @@ class BedrockConverseMessagesProcessor:
if _tool_calls:
assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model))
if (
isinstance(_assistant_content, list)
and len(assistant_content) > assistant_message_content_start
and assistant_message_block.get("cache_control") is not None
and not any("cachePoint" in block for block in assistant_content[assistant_message_content_start:])
):
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
assistant_message_block, block_type="content_block", model=model
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
msg_i += 1
assistant_content = _deduplicate_bedrock_content_blocks(assistant_content, "toolUse")
@ -4915,6 +4948,7 @@ def _bedrock_converse_messages_pt(
message=messages[msg_i],
assistant_continue_message=assistant_continue_message,
)
assistant_message_content_start: int = len(assistant_content)
_assistant_content = assistant_message_block.get("content", None)
thinking_blocks = cast(
list[ChatCompletionThinkingBlock] | None,
@ -4984,6 +5018,18 @@ def _bedrock_converse_messages_pt(
if _tool_calls:
assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model))
if (
isinstance(_assistant_content, list)
and len(assistant_content) > assistant_message_content_start
and assistant_message_block.get("cache_control") is not None
and not any("cachePoint" in block for block in assistant_content[assistant_message_content_start:])
):
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
assistant_message_block, block_type="content_block", model=model
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
msg_i += 1
assistant_content = _deduplicate_bedrock_content_blocks(assistant_content, "toolUse")

3
litellm/llms/anthropic/chat/transformation.py
View file

@ -1745,8 +1745,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
and system_message_block.get("cache_control") is not None
and not any(
content.get("cache_control") is not None
for content in message["content"]
if isinstance(content, dict)
for content in anthropic_system_message_list[content_start:]
)
):
anthropic_system_message_list[-1]["cache_control"] = system_message_block["cache_control"]

105
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -55,6 +55,73 @@ def test_anthropic_list_content_preserves_message_cache_control(role, block_cach
assert result[-1]["content"] == expected
@pytest.mark.parametrize("call_field", ["tool_calls", "function_call"])
def test_anthropic_list_message_cache_control_follows_tool_calls(call_field):
cache_control: Final = {"type": "ephemeral"}
call: Final = (
[{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}]
if call_field == "tool_calls"
else {"name": "lookup", "arguments": "{}"}
)
messages: Final = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": [{"type": "text", "text": "checking"}],
"cache_control": cache_control,
call_field: call,
},
]
result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic")
blocks: Final = result[-1]["content"]
assert blocks[0] == {"type": "text", "text": "checking"}
assert blocks[-1]["type"] == "tool_use"
assert blocks[-1]["cache_control"] == cache_control
@pytest.mark.parametrize("role", ["user", "assistant"])
def test_anthropic_list_message_cache_control_preserves_caller_block(role):
block: Final = (
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "document"}}
if role == "user"
else {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}
)
messages: Final = [
{"role": "user", "content": "hi"},
{"role": role, "content": [block], "cache_control": {"type": "ephemeral"}},
]
result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic")
assert "cache_control" not in block
assert result[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"}
assert result[-1]["content"][-1] is not block
@pytest.mark.parametrize("role", ["user", "assistant"])
def test_anthropic_skipped_list_block_does_not_suppress_message_cache_control(role):
skipped_block: Final = (
{"type": "unknown", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
if role == "user"
else {"type": "thinking", "thinking": "unsigned", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
)
messages: Final = [
{"role": "user", "content": "hi"},
{
"role": role,
"content": [skipped_block, {"type": "text", "text": "retained"}],
"cache_control": {"type": "ephemeral"},
},
]
result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic")
assert result[-1]["content"][-1] == {"type": "text", "text": "retained", "cache_control": {"type": "ephemeral"}}
assert all(block.get("type") != skipped_block["type"] for block in result[-1]["content"])
@pytest.mark.asyncio
@pytest.mark.parametrize("block_cache_control", [False, True])
async def test_bedrock_list_content_preserves_message_cache_control(block_cache_control):
@ -80,6 +147,44 @@ async def test_bedrock_list_content_preserves_message_cache_control(block_cache_
assert async_result[0]["content"] == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("block_cache_control", [False, True])
@pytest.mark.parametrize("with_tool_calls", [False, True])
async def test_bedrock_assistant_list_preserves_message_cache_control(block_cache_control, with_tool_calls):
content: Final = [
{
"type": "text",
"text": "checking",
**({"cache_control": {"type": "ephemeral"}} if block_cache_control else {}),
}
]
assistant_message: Final = {
"role": "assistant",
"content": content,
"cache_control": {"type": "ephemeral"},
**(
{"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}]}
if with_tool_calls
else {}
),
}
messages: Final = [{"role": "user", "content": "hi"}, assistant_message]
model: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
sync_result: Final = _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock")
async_result: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
messages=messages, model=model, llm_provider="bedrock"
)
for result in (sync_result, async_result):
blocks = result[-1]["content"]
assert sum("cachePoint" in block for block in blocks) == 1
if not block_cache_control:
assert blocks[-1] == {"cachePoint": {"type": "default"}}
if with_tool_calls:
assert "toolUse" in blocks[-2]
def _get_gemini_function_response_inline_data_parts(result):
assert isinstance(result, list), "expected Gemini parts list"
assert len(result) == 1, "multimodal function responses should stay in one part"

18
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -2409,6 +2409,24 @@ def test_translate_system_list_content_preserves_message_cache_control(block_cac
assert result == expected
def test_translate_system_skipped_list_block_does_not_suppress_message_cache_control():
messages: Final = [
{
"role": "system",
"content": [
{"type": "text", "text": "", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
{"type": "text", "text": "retained"},
],
"cache_control": {"type": "ephemeral"},
},
{"role": "user", "content": "hi"},
]
result: Final = AnthropicConfig().translate_system_message(messages)
assert result == [{"type": "text", "text": "retained", "cache_control": {"type": "ephemeral"}}]
# ============ Dynamic max_tokens Tests ============