scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(bedrock): preserve cache_control ttl on message-level cache points (#32538)
Some checks are pending
CodSpeed Benchmarks / benchmarks (push) Waiting to run
Details
GitHub Actions Security Analysis / zizmor (push) Waiting to run
Details

Browse source 
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-07-08 14:00:24 -07:00 • committed by GitHub
parent e9e30dffb6
commit 0f1e29b334
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 89 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -4377,6 +4377,7 @@ class BedrockConverseMessagesProcessor:
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(OpenAIMessageContentListBlock, element),
block_type="content_block",
model=model,
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
@ -4384,7 +4385,7 @@ class BedrockConverseMessagesProcessor:
elif message_block["content"] and isinstance(message_block["content"], str):
_part = BedrockContentBlock(text=messages[msg_i]["content"])
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
message_block, block_type="content_block"
message_block, block_type="content_block", model=model
)
user_content.append(_part)
if _cache_point_block is not None:
@ -4509,6 +4510,7 @@ class BedrockConverseMessagesProcessor:
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(OpenAIMessageContentListBlock, element),
block_type="content_block",
model=model,
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
@ -4520,7 +4522,7 @@ class BedrockConverseMessagesProcessor:
# If content is empty/whitespace, skip it (don't add a placeholder)
# Add cache point block for assistant string content
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
assistant_message_block, block_type="content_block", model=model
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
@ -4745,6 +4747,7 @@ def _bedrock_converse_messages_pt(
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(OpenAIMessageContentListBlock, element),
block_type="content_block",
model=model,
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
@ -4752,7 +4755,7 @@ def _bedrock_converse_messages_pt(
elif message_block["content"] and isinstance(message_block["content"], str):
_part = BedrockContentBlock(text=messages[msg_i]["content"])
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
message_block, block_type="content_block"
message_block, block_type="content_block", model=model
)
user_content.append(_part)
if _cache_point_block is not None:
@ -4882,6 +4885,7 @@ def _bedrock_converse_messages_pt(
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(OpenAIMessageContentListBlock, element),
block_type="content_block",
model=model,
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
@ -4892,7 +4896,7 @@ def _bedrock_converse_messages_pt(
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
assistant_message_block, block_type="content_block", model=model
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)

81
tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -3085,3 +3085,84 @@ def test_bedrock_converse_messages_pt_document_rejects_url_source():
_bedrock_converse_messages_pt(
messages, "anthropic.claude-sonnet-4-6", "bedrock"
)
def _collect_cache_points(blocks):
return [
block["cachePoint"]
for message in blocks
for block in message["content"]
if "cachePoint" in block
]
@pytest.mark.parametrize(
"messages",
[
[
{
"role": "user",
"content": [
{
"type": "text",
"text": "conversation history",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
}
],
},
],
[
{"role": "user", "content": "hello"},
{
"role": "assistant",
"content": [
{
"type": "text",
"text": "assistant reply",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
}
],
},
],
],
)
def test_bedrock_converse_message_level_cache_point_preserves_ttl(messages):
"""
Regression for https://github.com/BerriAI/litellm/issues/32154: message-level
cache_control ttl was silently dropped because the message-level
_get_cache_point_block call sites never passed model=, so multi-turn prefixes
fell back to the 5m default while the system prompt kept 1h, churning the
cache every turn on models like Opus 4.8.
"""
result = _bedrock_converse_messages_pt(
messages=messages,
model="eu.anthropic.claude-opus-4-8",
llm_provider="bedrock",
)
cache_points = _collect_cache_points(result)
assert cache_points == [{"type": "default", "ttl": "1h"}]
@pytest.mark.asyncio
async def test_bedrock_converse_message_level_cache_point_preserves_ttl_async():
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "conversation history",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
}
],
},
]
result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
messages=messages,
model="eu.anthropic.claude-opus-4-8",
llm_provider="bedrock",
)
assert _collect_cache_points(result) == [{"type": "default", "ttl": "1h"}]