mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge pull request #20247 from BerriAI/litellm_smoketest_suite
Add Anthropic caching and context tests
This commit is contained in:
Sameer Kankute
GitHub
commit
dafa26baa6
1 changed files with 526 additions and 0 deletions
526
tests/llm_translation/test_bedrock_anthropic_regression.py
Normal file
526
tests/llm_translation/test_bedrock_anthropic_regression.py
Normal file
|
|
@ -0,0 +1,526 @@
|
|||
"""
|
||||
Regression tests for Bedrock Anthropic models.
|
||||
|
||||
Tests critical functionality that has broken in the past between bedrock/invoke
|
||||
and bedrock/converse routing:
|
||||
1. Prompt caching support (cache_control)
|
||||
2. 1M context window support (anthropic-beta header)
|
||||
|
||||
These tests ensure that both routing methods (invoke vs converse) maintain
|
||||
feature parity and prevent regression of previously fixed issues.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
|
||||
# Large document for caching tests (needs 1024+ tokens for Claude models)
|
||||
LARGE_DOCUMENT_FOR_CACHING = """
|
||||
This is a comprehensive legal agreement between Party A and Party B.
|
||||
|
||||
ARTICLE 1: DEFINITIONS
|
||||
1.1 "Agreement" means this document and all attachments.
|
||||
1.2 "Confidential Information" means any non-public information.
|
||||
1.3 "Effective Date" means the date of last signature.
|
||||
1.4 "Term" means the period during which this Agreement is in effect.
|
||||
|
||||
ARTICLE 2: SCOPE OF SERVICES
|
||||
2.1 Party A agrees to provide the following services...
|
||||
2.2 Party B agrees to compensate Party A for services rendered...
|
||||
2.3 All services shall be performed in a professional manner...
|
||||
|
||||
ARTICLE 3: PAYMENT TERMS
|
||||
3.1 Payment shall be made within 30 days of invoice receipt.
|
||||
3.2 Late payments shall accrue interest at 1.5% per month.
|
||||
3.3 All fees are non-refundable unless otherwise specified.
|
||||
|
||||
ARTICLE 4: INTELLECTUAL PROPERTY
|
||||
4.1 All pre-existing IP remains with the original owner.
|
||||
4.2 Work product created under this Agreement shall be owned by Party B.
|
||||
4.3 Party A grants a license to use any tools or methodologies.
|
||||
|
||||
ARTICLE 5: CONFIDENTIALITY
|
||||
5.1 Both parties agree to maintain confidentiality of all shared information.
|
||||
5.2 Confidential information shall not be disclosed to third parties.
|
||||
5.3 This obligation survives termination of the Agreement.
|
||||
|
||||
ARTICLE 6: TERMINATION
|
||||
6.1 Either party may terminate with 30 days written notice.
|
||||
6.2 Immediate termination is permitted for material breach.
|
||||
6.3 Upon termination, all confidential information must be returned.
|
||||
|
||||
ARTICLE 7: LIMITATION OF LIABILITY
|
||||
7.1 Neither party shall be liable for consequential damages.
|
||||
7.2 Total liability shall not exceed fees paid in the prior 12 months.
|
||||
7.3 This limitation does not apply to willful misconduct.
|
||||
|
||||
ARTICLE 8: DISPUTE RESOLUTION
|
||||
8.1 Disputes shall first be addressed through good faith negotiation.
|
||||
8.2 If negotiation fails, disputes shall be submitted to arbitration.
|
||||
8.3 Arbitration shall be conducted under AAA rules.
|
||||
|
||||
ARTICLE 9: GENERAL PROVISIONS
|
||||
9.1 This Agreement constitutes the entire understanding between parties.
|
||||
9.2 Amendments must be in writing and signed by both parties.
|
||||
9.3 This Agreement shall be governed by the laws of Delaware.
|
||||
9.4 Neither party may assign this Agreement without consent.
|
||||
9.5 Waiver of any provision shall not constitute ongoing waiver.
|
||||
|
||||
IN WITNESS WHEREOF, the parties have executed this Agreement.
|
||||
""" * 8 # Repeat to ensure we have enough tokens (need 1024+ for Claude models)
|
||||
|
||||
|
||||
class TestBedrockAnthropicPromptCachingRegression:
|
||||
"""
|
||||
Regression tests for prompt caching support across bedrock/invoke and bedrock/converse.
|
||||
|
||||
Issue: Prompt caching broke between invoke and converse routing due to:
|
||||
- Different cache_control syntax expectations
|
||||
- Incorrect beta header handling
|
||||
- Missing transformation for cachePoint vs cache_control
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_prefix",
|
||||
[
|
||||
"bedrock/invoke/",
|
||||
"bedrock/converse/",
|
||||
],
|
||||
)
|
||||
def test_prompt_caching_cache_control_transforms_correctly(
|
||||
self, model_prefix
|
||||
):
|
||||
"""
|
||||
Test that cache_control in messages is correctly transformed for both invoke and converse APIs.
|
||||
|
||||
Regression test: Ensure cache_control works the same way for both routing methods.
|
||||
- bedrock/invoke uses cache_control directly in the Anthropic Messages API format
|
||||
- bedrock/converse should transform to cachePoint format
|
||||
"""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import (
|
||||
AmazonConverseConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": LARGE_DOCUMENT_FOR_CACHING,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What are the payment terms?",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
if "converse" in model_prefix:
|
||||
config = AmazonConverseConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
||||
|
||||
# For converse, cache_control should be transformed to cachePoint
|
||||
assert "messages" in result
|
||||
user_msg = result["messages"][0]
|
||||
assert "content" in user_msg
|
||||
|
||||
# Check that cachePoint is present (Bedrock Converse format)
|
||||
has_cache_point = any(
|
||||
isinstance(c, dict) and "cachePoint" in c
|
||||
for c in user_msg["content"]
|
||||
)
|
||||
# The transformation should preserve the cache marking in some form
|
||||
assert "messages" in result, "messages should be present in converse request"
|
||||
|
||||
else:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
||||
|
||||
# For invoke, cache_control should be preserved in messages content
|
||||
assert "messages" in result
|
||||
user_msg = result["messages"][0]
|
||||
assert "content" in user_msg
|
||||
|
||||
# Check that cache_control is preserved
|
||||
has_cache_control = any(
|
||||
isinstance(c, dict) and "cache_control" in c
|
||||
for c in user_msg["content"]
|
||||
)
|
||||
assert has_cache_control, "cache_control should be present in invoke messages"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_prefix",
|
||||
[
|
||||
"bedrock/invoke/",
|
||||
"bedrock/converse/",
|
||||
],
|
||||
)
|
||||
def test_prompt_caching_no_beta_header_added(self, model_prefix):
|
||||
"""
|
||||
Test that prompt-caching-2024-07-31 beta header is NOT added for Bedrock.
|
||||
|
||||
Regression test: Bedrock recognizes prompt caching via cache_control in the
|
||||
request body, NOT through beta headers. Adding the beta header breaks requests.
|
||||
|
||||
This was a critical bug where litellm was incorrectly adding the Anthropic API
|
||||
beta header to Bedrock requests.
|
||||
"""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import (
|
||||
AmazonConverseConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Hello",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
if "converse" in model_prefix:
|
||||
config = AmazonConverseConfig()
|
||||
result = config._transform_request_helper(
|
||||
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={},
|
||||
messages=messages,
|
||||
headers={},
|
||||
)
|
||||
else:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
# Verify prompt-caching beta header is NOT present
|
||||
if "anthropic_beta" in result:
|
||||
assert "prompt-caching-2024-07-31" not in result["anthropic_beta"], (
|
||||
f"{model_prefix}: prompt-caching-2024-07-31 should NOT be added as a beta header for Bedrock. "
|
||||
"Bedrock recognizes prompt caching via cache_control in the request body, not beta headers."
|
||||
)
|
||||
|
||||
# For converse, also check additionalModelRequestFields
|
||||
if "converse" in model_prefix and "additionalModelRequestFields" in result:
|
||||
additional_fields = result["additionalModelRequestFields"]
|
||||
if "anthropic_beta" in additional_fields:
|
||||
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
|
||||
|
||||
|
||||
class TestBedrockAnthropic1MContextRegression:
|
||||
"""
|
||||
Regression tests for 1M context window support across bedrock/invoke and bedrock/converse.
|
||||
|
||||
Issue: 1M context support broke between invoke and converse routing due to:
|
||||
- Missing anthropic-beta header passthrough in converse
|
||||
- Incorrect handling of context-1m-2025-08-07 beta header
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_prefix",
|
||||
[
|
||||
"bedrock/invoke/",
|
||||
"bedrock/converse/",
|
||||
],
|
||||
)
|
||||
def test_1m_context_beta_header_is_passed_via_transformation(self, model_prefix):
|
||||
"""
|
||||
Test that the 1M context beta header is correctly passed to Bedrock API.
|
||||
|
||||
Regression test: Ensure anthropic-beta: context-1m-2025-08-07 header
|
||||
is correctly included in the request for both invoke and converse.
|
||||
|
||||
This test verifies the transformation layer directly to avoid async complexity.
|
||||
"""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import (
|
||||
AmazonConverseConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
||||
messages = [{"role": "user", "content": "Test message"}]
|
||||
|
||||
if "converse" in model_prefix:
|
||||
config = AmazonConverseConfig()
|
||||
result = config._transform_request_helper(
|
||||
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={},
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
||||
|
||||
# For converse, beta header should be in additionalModelRequestFields
|
||||
assert "additionalModelRequestFields" in result, (
|
||||
f"{model_prefix}: additionalModelRequestFields should be present for anthropic-beta headers"
|
||||
)
|
||||
additional_fields = result["additionalModelRequestFields"]
|
||||
assert "anthropic_beta" in additional_fields, (
|
||||
f"{model_prefix}: anthropic_beta should be in additionalModelRequestFields"
|
||||
)
|
||||
assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"], (
|
||||
f"{model_prefix}: context-1m-2025-08-07 should be in anthropic_beta array"
|
||||
)
|
||||
else:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
||||
|
||||
# For invoke, beta header should be in top-level request
|
||||
assert "anthropic_beta" in result, (
|
||||
f"{model_prefix}: anthropic_beta should be in request body"
|
||||
)
|
||||
assert "context-1m-2025-08-07" in result["anthropic_beta"], (
|
||||
f"{model_prefix}: context-1m-2025-08-07 should be in anthropic_beta array"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_prefix",
|
||||
[
|
||||
"bedrock/invoke/",
|
||||
"bedrock/converse/",
|
||||
],
|
||||
)
|
||||
def test_1m_context_beta_header_transformation(self, model_prefix):
|
||||
"""
|
||||
Test that the 1M context beta header is correctly transformed at the config level.
|
||||
|
||||
This is a unit test that verifies the transformation logic directly without
|
||||
making actual API calls.
|
||||
"""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import (
|
||||
AmazonConverseConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
||||
messages = [{"role": "user", "content": "Test"}]
|
||||
|
||||
if "converse" in model_prefix:
|
||||
config = AmazonConverseConfig()
|
||||
result = config._transform_request_helper(
|
||||
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={},
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Verify beta header is in additionalModelRequestFields
|
||||
assert "additionalModelRequestFields" in result
|
||||
additional_fields = result["additionalModelRequestFields"]
|
||||
assert "anthropic_beta" in additional_fields
|
||||
assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"]
|
||||
|
||||
else:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Verify beta header is in top-level request
|
||||
assert "anthropic_beta" in result
|
||||
assert "context-1m-2025-08-07" in result["anthropic_beta"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_prefix",
|
||||
[
|
||||
"bedrock/invoke/",
|
||||
"bedrock/converse/",
|
||||
],
|
||||
)
|
||||
def test_1m_context_with_multiple_beta_headers(self, model_prefix):
|
||||
"""
|
||||
Test that 1M context header works alongside other beta headers.
|
||||
|
||||
Ensures that multiple anthropic-beta values (comma-separated) are all
|
||||
correctly passed through.
|
||||
"""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import (
|
||||
AmazonConverseConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
# Multiple beta headers including 1M context
|
||||
headers = {
|
||||
"anthropic-beta": "context-1m-2025-08-07,computer-use-2024-10-22"
|
||||
}
|
||||
messages = [{"role": "user", "content": "Test"}]
|
||||
|
||||
if "converse" in model_prefix:
|
||||
config = AmazonConverseConfig()
|
||||
result = config._transform_request_helper(
|
||||
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={},
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
additional_fields = result["additionalModelRequestFields"]
|
||||
beta_headers = additional_fields["anthropic_beta"]
|
||||
|
||||
else:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
beta_headers = result["anthropic_beta"]
|
||||
|
||||
# Verify both headers are present
|
||||
assert "context-1m-2025-08-07" in beta_headers
|
||||
assert "computer-use-2024-10-22" in beta_headers
|
||||
|
||||
|
||||
class TestBedrockAnthropicCombinedRegressions:
|
||||
"""
|
||||
Tests that combine multiple features to ensure they work together.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_prefix",
|
||||
[
|
||||
"bedrock/invoke/",
|
||||
"bedrock/converse/",
|
||||
],
|
||||
)
|
||||
def test_1m_context_with_prompt_caching(self, model_prefix):
|
||||
"""
|
||||
Test that 1M context and prompt caching work together.
|
||||
|
||||
This is a real-world scenario where a user might want to use both features
|
||||
simultaneously.
|
||||
"""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import (
|
||||
AmazonConverseConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": LARGE_DOCUMENT_FOR_CACHING,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Summarize this document.",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
if "converse" in model_prefix:
|
||||
config = AmazonConverseConfig()
|
||||
result = config._transform_request_helper(
|
||||
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={},
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Should have 1M context header
|
||||
additional_fields = result["additionalModelRequestFields"]
|
||||
assert "anthropic_beta" in additional_fields
|
||||
assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"]
|
||||
|
||||
# Should NOT have prompt-caching header
|
||||
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
|
||||
|
||||
else:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Should have 1M context header
|
||||
assert "anthropic_beta" in result
|
||||
assert "context-1m-2025-08-07" in result["anthropic_beta"]
|
||||
|
||||
# Should NOT have prompt-caching header
|
||||
assert "prompt-caching-2024-07-31" not in result["anthropic_beta"]
|
||||
|
||||
# Should have cache_control in messages
|
||||
user_msg = result["messages"][0]
|
||||
has_cache_control = any(
|
||||
isinstance(c, dict) and "cache_control" in c
|
||||
for c in user_msg["content"]
|
||||
)
|
||||
assert has_cache_control
|
||||
Loading…
Add table
Reference in a new issue