mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
fix(bedrock): strip scope from cache_control for Anthropic messages
Bedrock does not support the scope field in cache_control (e.g. 'global' for cross-request caching). Only type and ttl are supported per AWS docs. - Remove scope from cache_control in both system and messages - Extend _remove_ttl_from_cache_control to process system blocks - Add test for scope removal Made-with: Cursor
This commit is contained in:
parent
1bb713bc7b
commit
cc989b1171
2 changed files with 76 additions and 15 deletions
|
|
@ -118,10 +118,13 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
self, anthropic_messages_request: Dict, model: Optional[str] = None
|
||||
) -> None:
|
||||
"""
|
||||
Remove `ttl` field from cache_control in messages.
|
||||
Bedrock doesn't support the ttl field in cache_control.
|
||||
Remove unsupported fields from cache_control for Bedrock.
|
||||
|
||||
Update: Bedock supports `5m` and `1h` for Claude 4.5 models.
|
||||
Bedrock only supports `type` and `ttl` in cache_control. It does NOT support:
|
||||
- `scope` (e.g., "global") - always removed
|
||||
- `ttl` - removed for older models; Claude 4.5+ supports "5m" and "1h"
|
||||
|
||||
Processes both `system` and `messages` content blocks.
|
||||
|
||||
Args:
|
||||
anthropic_messages_request: The request dictionary to modify in-place
|
||||
|
|
@ -131,23 +134,36 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if model:
|
||||
is_claude_4_5 = self._is_claude_4_5_on_bedrock(model)
|
||||
|
||||
def _sanitize_cache_control(cache_control: dict) -> None:
|
||||
if not isinstance(cache_control, dict):
|
||||
return
|
||||
# Bedrock doesn't support scope (e.g., "global" for cross-request caching)
|
||||
cache_control.pop("scope", None)
|
||||
# Remove ttl for models that don't support it
|
||||
if "ttl" in cache_control:
|
||||
ttl = cache_control["ttl"]
|
||||
if is_claude_4_5 and ttl in ["5m", "1h"]:
|
||||
return
|
||||
cache_control.pop("ttl", None)
|
||||
|
||||
def _process_content_list(content: list) -> None:
|
||||
for item in content:
|
||||
if isinstance(item, dict) and "cache_control" in item:
|
||||
_sanitize_cache_control(item["cache_control"])
|
||||
|
||||
# Process system (list of content blocks)
|
||||
if "system" in anthropic_messages_request:
|
||||
system = anthropic_messages_request["system"]
|
||||
if isinstance(system, list):
|
||||
_process_content_list(system)
|
||||
|
||||
# Process messages
|
||||
if "messages" in anthropic_messages_request:
|
||||
for message in anthropic_messages_request["messages"]:
|
||||
if isinstance(message, dict) and "content" in message:
|
||||
content = message["content"]
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if isinstance(item, dict) and "cache_control" in item:
|
||||
cache_control = item["cache_control"]
|
||||
if (
|
||||
isinstance(cache_control, dict)
|
||||
and "ttl" in cache_control
|
||||
):
|
||||
ttl = cache_control["ttl"]
|
||||
if is_claude_4_5 and ttl in ["5m", "1h"]:
|
||||
continue
|
||||
|
||||
cache_control.pop("ttl", None)
|
||||
_process_content_list(content)
|
||||
|
||||
def _supports_extended_thinking_on_bedrock(self, model: str) -> bool:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -178,3 +178,48 @@ def test_remove_ttl_from_cache_control():
|
|||
request5 = {}
|
||||
cfg._remove_ttl_from_cache_control(request5)
|
||||
assert request5 == {}
|
||||
|
||||
|
||||
def test_remove_scope_from_cache_control():
|
||||
"""Ensure scope field is removed from cache_control for Bedrock (not supported)."""
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
|
||||
# Test case 1: System with cache_control containing scope
|
||||
request = {
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant.",
|
||||
"cache_control": {
|
||||
"type": "ephemeral",
|
||||
"scope": "global",
|
||||
},
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Hello",
|
||||
"cache_control": {
|
||||
"type": "ephemeral",
|
||||
"scope": "global",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
cfg._remove_ttl_from_cache_control(request)
|
||||
|
||||
# Verify scope is removed from system
|
||||
assert "scope" not in request["system"][0]["cache_control"]
|
||||
assert request["system"][0]["cache_control"]["type"] == "ephemeral"
|
||||
|
||||
# Verify scope is removed from messages
|
||||
assert "scope" not in request["messages"][0]["content"][0]["cache_control"]
|
||||
assert request["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue