mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(bedrock): preserve cache_control for ARN models in /v1/messages adapter (#29823)
* fix(bedrock): preserve cache_control for ARN models in /v1/messages adapter Bedrock Application Inference Profile ARNs contain neither "anthropic" nor "claude", so is_anthropic_claude_model could not detect them and the /v1/messages adapter silently dropped cache_control during the Anthropic to OpenAI translation. Prompt caching never activated for these models, while the same profile cached correctly through /v1/chat/completions. Add an is_bedrock_arn_model check scoped to _add_cache_control_if_applicable so cache_control is preserved for ARN-based models without broadening the shared is_anthropic_claude_model helper, which also drives thinking translation. Fixes #26625 * refactor(bedrock): match :bedrock: ARN service field in is_bedrock_arn_model Tighten the ARN detection so it pins "bedrock" to the colon-delimited service field of the ARN rather than matching the substring anywhere. This avoids a false positive for another service's ARN whose resource name merely contains "bedrock" (e.g. arn:aws:sagemaker:...:endpoint/my-bedrock-transcriber).
This commit is contained in:
parent
f444539ea9
commit
ead8a708bb
2 changed files with 90 additions and 1 deletions
|
|
@ -332,7 +332,14 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if isinstance(source, dict)
|
||||
else getattr(source, "cache_control", None)
|
||||
)
|
||||
if cache_control and model and self.is_anthropic_claude_model(model):
|
||||
if (
|
||||
cache_control
|
||||
and model
|
||||
and (
|
||||
self.is_anthropic_claude_model(model)
|
||||
or self.is_bedrock_arn_model(model)
|
||||
)
|
||||
):
|
||||
# TypedDict objects support dict operations at runtime
|
||||
# Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
|
||||
if isinstance(target, dict):
|
||||
|
|
@ -752,6 +759,20 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
model_lower = model.lower()
|
||||
return "anthropic" in model_lower or "claude" in model_lower
|
||||
|
||||
@staticmethod
|
||||
def is_bedrock_arn_model(model: str) -> bool:
|
||||
"""
|
||||
Check if the model string is a Bedrock ARN, such as an Application
|
||||
Inference Profile (e.g. arn:aws:bedrock:us-east-1:123:application-inference-profile/id).
|
||||
|
||||
These ARNs contain neither "anthropic" nor "claude", so is_anthropic_claude_model
|
||||
cannot identify them even though, on the /v1/messages endpoint, they point at Claude.
|
||||
Match ":bedrock:" in the ARN service field so another service's ARN that merely names
|
||||
bedrock in a resource (arn:aws:sagemaker:.../my-bedrock-endpoint) is not matched.
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
return "arn:" in model_lower and ":bedrock:" in model_lower
|
||||
|
||||
@staticmethod
|
||||
def translate_thinking_for_model(
|
||||
thinking: Dict[str, Any],
|
||||
|
|
|
|||
|
|
@ -1295,6 +1295,12 @@ CACHE_CONTROL_BEDROCK_CONVERSE_MODEL = (
|
|||
"bedrock/converse/global.anthropic.claude-opus-4-5-20251101-v1:0"
|
||||
)
|
||||
CACHE_CONTROL_NON_ANTHROPIC_MODEL = "gpt-4"
|
||||
# Bedrock Application Inference Profile ARN: the string contains neither
|
||||
# "anthropic" nor "claude", so the model can only be recognized via its ARN shape
|
||||
CACHE_CONTROL_BEDROCK_ARN_MODEL = (
|
||||
"bedrock/converse/arn:aws:bedrock:us-east-1:123456789012:"
|
||||
"application-inference-profile/abcdef123456"
|
||||
)
|
||||
|
||||
|
||||
def test_should_add_cache_control_for_anthropic_model():
|
||||
|
|
@ -1411,6 +1417,68 @@ def test_cache_control_not_preserved_for_non_claude_model():
|
|||
assert "cache_control" not in result[0]["content"][0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, expected",
|
||||
[
|
||||
(CACHE_CONTROL_BEDROCK_ARN_MODEL, True),
|
||||
(
|
||||
"arn:aws-us-gov:bedrock:us-gov-west-1:123:application-inference-profile/x",
|
||||
True,
|
||||
),
|
||||
("bedrock/amazon.titan-text-express-v1", False),
|
||||
("arn:aws:sagemaker:us-east-1:123:endpoint/my-endpoint", False),
|
||||
("arn:aws:sagemaker:us-east-1:123:endpoint/my-bedrock-transcriber", False),
|
||||
(CACHE_CONTROL_NON_ANTHROPIC_MODEL, False),
|
||||
],
|
||||
)
|
||||
def test_is_bedrock_arn_model(model, expected):
|
||||
"""is_bedrock_arn_model requires an ARN with bedrock in the service field, not just anywhere."""
|
||||
assert LiteLLMAnthropicMessagesAdapter.is_bedrock_arn_model(model) is expected
|
||||
|
||||
|
||||
def test_cache_control_preserved_for_bedrock_arn_inference_profile():
|
||||
"""
|
||||
Regression for https://github.com/BerriAI/litellm/issues/26625
|
||||
|
||||
Bedrock Application Inference Profile ARNs hide the underlying Claude model
|
||||
name, so cache_control must still be preserved through the /v1/messages adapter.
|
||||
"""
|
||||
anthropic_messages = [
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "This is cached content",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
result = adapter.translate_anthropic_messages_to_openai(
|
||||
messages=anthropic_messages, model=CACHE_CONTROL_BEDROCK_ARN_MODEL
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_cache_control_fix_does_not_broaden_claude_detection():
|
||||
"""
|
||||
The cache_control fix is scoped to _add_cache_control_if_applicable; it must not
|
||||
make is_anthropic_claude_model treat ARN profiles as Claude, which would route
|
||||
thinking params through unmodified and break non-Claude Bedrock profiles.
|
||||
"""
|
||||
assert (
|
||||
LiteLLMAnthropicMessagesAdapter.is_anthropic_claude_model(
|
||||
CACHE_CONTROL_BEDROCK_ARN_MODEL
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_image_content_for_claude():
|
||||
"""Cache control should be preserved in image content for Claude models."""
|
||||
anthropic_messages = [
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue