diff --git a/tests/llm_translation/test_bedrock_anthropic_regression.py b/tests/llm_translation/test_bedrock_anthropic_regression.py new file mode 100644 index 00000000000..df8755ba1ad --- /dev/null +++ b/tests/llm_translation/test_bedrock_anthropic_regression.py @@ -0,0 +1,526 @@ +""" +Regression tests for Bedrock Anthropic models. + +Tests critical functionality that has broken in the past between bedrock/invoke +and bedrock/converse routing: +1. Prompt caching support (cache_control) +2. 1M context window support (anthropic-beta header) + +These tests ensure that both routing methods (invoke vs converse) maintain +feature parity and prevent regression of previously fixed issues. +""" + +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm import completion + + +# Large document for caching tests (needs 1024+ tokens for Claude models) +LARGE_DOCUMENT_FOR_CACHING = """ +This is a comprehensive legal agreement between Party A and Party B. + +ARTICLE 1: DEFINITIONS +1.1 "Agreement" means this document and all attachments. +1.2 "Confidential Information" means any non-public information. +1.3 "Effective Date" means the date of last signature. +1.4 "Term" means the period during which this Agreement is in effect. + +ARTICLE 2: SCOPE OF SERVICES +2.1 Party A agrees to provide the following services... +2.2 Party B agrees to compensate Party A for services rendered... +2.3 All services shall be performed in a professional manner... + +ARTICLE 3: PAYMENT TERMS +3.1 Payment shall be made within 30 days of invoice receipt. +3.2 Late payments shall accrue interest at 1.5% per month. +3.3 All fees are non-refundable unless otherwise specified. + +ARTICLE 4: INTELLECTUAL PROPERTY +4.1 All pre-existing IP remains with the original owner. +4.2 Work product created under this Agreement shall be owned by Party B. +4.3 Party A grants a license to use any tools or methodologies. + +ARTICLE 5: CONFIDENTIALITY +5.1 Both parties agree to maintain confidentiality of all shared information. +5.2 Confidential information shall not be disclosed to third parties. +5.3 This obligation survives termination of the Agreement. + +ARTICLE 6: TERMINATION +6.1 Either party may terminate with 30 days written notice. +6.2 Immediate termination is permitted for material breach. +6.3 Upon termination, all confidential information must be returned. + +ARTICLE 7: LIMITATION OF LIABILITY +7.1 Neither party shall be liable for consequential damages. +7.2 Total liability shall not exceed fees paid in the prior 12 months. +7.3 This limitation does not apply to willful misconduct. + +ARTICLE 8: DISPUTE RESOLUTION +8.1 Disputes shall first be addressed through good faith negotiation. +8.2 If negotiation fails, disputes shall be submitted to arbitration. +8.3 Arbitration shall be conducted under AAA rules. + +ARTICLE 9: GENERAL PROVISIONS +9.1 This Agreement constitutes the entire understanding between parties. +9.2 Amendments must be in writing and signed by both parties. +9.3 This Agreement shall be governed by the laws of Delaware. +9.4 Neither party may assign this Agreement without consent. +9.5 Waiver of any provision shall not constitute ongoing waiver. + +IN WITNESS WHEREOF, the parties have executed this Agreement. +""" * 8 # Repeat to ensure we have enough tokens (need 1024+ for Claude models) + + +class TestBedrockAnthropicPromptCachingRegression: + """ + Regression tests for prompt caching support across bedrock/invoke and bedrock/converse. + + Issue: Prompt caching broke between invoke and converse routing due to: + - Different cache_control syntax expectations + - Incorrect beta header handling + - Missing transformation for cachePoint vs cache_control + """ + + @pytest.mark.parametrize( + "model_prefix", + [ + "bedrock/invoke/", + "bedrock/converse/", + ], + ) + def test_prompt_caching_cache_control_transforms_correctly( + self, model_prefix + ): + """ + Test that cache_control in messages is correctly transformed for both invoke and converse APIs. + + Regression test: Ensure cache_control works the same way for both routing methods. + - bedrock/invoke uses cache_control directly in the Anthropic Messages API format + - bedrock/converse should transform to cachePoint format + """ + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": LARGE_DOCUMENT_FOR_CACHING, + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "text", + "text": "What are the payment terms?", + }, + ], + }, + ] + + if "converse" in model_prefix: + config = AmazonConverseConfig() + result = config.transform_request( + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}") + + # For converse, cache_control should be transformed to cachePoint + assert "messages" in result + user_msg = result["messages"][0] + assert "content" in user_msg + + # Check that cachePoint is present (Bedrock Converse format) + has_cache_point = any( + isinstance(c, dict) and "cachePoint" in c + for c in user_msg["content"] + ) + # The transformation should preserve the cache marking in some form + assert "messages" in result, "messages should be present in converse request" + + else: + config = AmazonAnthropicClaudeConfig() + result = config.transform_request( + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}") + + # For invoke, cache_control should be preserved in messages content + assert "messages" in result + user_msg = result["messages"][0] + assert "content" in user_msg + + # Check that cache_control is preserved + has_cache_control = any( + isinstance(c, dict) and "cache_control" in c + for c in user_msg["content"] + ) + assert has_cache_control, "cache_control should be present in invoke messages" + + @pytest.mark.parametrize( + "model_prefix", + [ + "bedrock/invoke/", + "bedrock/converse/", + ], + ) + def test_prompt_caching_no_beta_header_added(self, model_prefix): + """ + Test that prompt-caching-2024-07-31 beta header is NOT added for Bedrock. + + Regression test: Bedrock recognizes prompt caching via cache_control in the + request body, NOT through beta headers. Adding the beta header breaks requests. + + This was a critical bug where litellm was incorrectly adding the Anthropic API + beta header to Bedrock requests. + """ + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + if "converse" in model_prefix: + config = AmazonConverseConfig() + result = config._transform_request_helper( + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + system_content_blocks=[], + optional_params={}, + messages=messages, + headers={}, + ) + else: + config = AmazonAnthropicClaudeConfig() + result = config.transform_request( + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify prompt-caching beta header is NOT present + if "anthropic_beta" in result: + assert "prompt-caching-2024-07-31" not in result["anthropic_beta"], ( + f"{model_prefix}: prompt-caching-2024-07-31 should NOT be added as a beta header for Bedrock. " + "Bedrock recognizes prompt caching via cache_control in the request body, not beta headers." + ) + + # For converse, also check additionalModelRequestFields + if "converse" in model_prefix and "additionalModelRequestFields" in result: + additional_fields = result["additionalModelRequestFields"] + if "anthropic_beta" in additional_fields: + assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"] + + +class TestBedrockAnthropic1MContextRegression: + """ + Regression tests for 1M context window support across bedrock/invoke and bedrock/converse. + + Issue: 1M context support broke between invoke and converse routing due to: + - Missing anthropic-beta header passthrough in converse + - Incorrect handling of context-1m-2025-08-07 beta header + """ + + @pytest.mark.parametrize( + "model_prefix", + [ + "bedrock/invoke/", + "bedrock/converse/", + ], + ) + def test_1m_context_beta_header_is_passed_via_transformation(self, model_prefix): + """ + Test that the 1M context beta header is correctly passed to Bedrock API. + + Regression test: Ensure anthropic-beta: context-1m-2025-08-07 header + is correctly included in the request for both invoke and converse. + + This test verifies the transformation layer directly to avoid async complexity. + """ + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + + headers = {"anthropic-beta": "context-1m-2025-08-07"} + messages = [{"role": "user", "content": "Test message"}] + + if "converse" in model_prefix: + config = AmazonConverseConfig() + result = config._transform_request_helper( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + system_content_blocks=[], + optional_params={}, + messages=messages, + headers=headers, + ) + + print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}") + + # For converse, beta header should be in additionalModelRequestFields + assert "additionalModelRequestFields" in result, ( + f"{model_prefix}: additionalModelRequestFields should be present for anthropic-beta headers" + ) + additional_fields = result["additionalModelRequestFields"] + assert "anthropic_beta" in additional_fields, ( + f"{model_prefix}: anthropic_beta should be in additionalModelRequestFields" + ) + assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"], ( + f"{model_prefix}: context-1m-2025-08-07 should be in anthropic_beta array" + ) + else: + config = AmazonAnthropicClaudeConfig() + result = config.transform_request( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers=headers, + ) + + print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}") + + # For invoke, beta header should be in top-level request + assert "anthropic_beta" in result, ( + f"{model_prefix}: anthropic_beta should be in request body" + ) + assert "context-1m-2025-08-07" in result["anthropic_beta"], ( + f"{model_prefix}: context-1m-2025-08-07 should be in anthropic_beta array" + ) + + @pytest.mark.parametrize( + "model_prefix", + [ + "bedrock/invoke/", + "bedrock/converse/", + ], + ) + def test_1m_context_beta_header_transformation(self, model_prefix): + """ + Test that the 1M context beta header is correctly transformed at the config level. + + This is a unit test that verifies the transformation logic directly without + making actual API calls. + """ + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + + headers = {"anthropic-beta": "context-1m-2025-08-07"} + messages = [{"role": "user", "content": "Test"}] + + if "converse" in model_prefix: + config = AmazonConverseConfig() + result = config._transform_request_helper( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + system_content_blocks=[], + optional_params={}, + messages=messages, + headers=headers, + ) + + # Verify beta header is in additionalModelRequestFields + assert "additionalModelRequestFields" in result + additional_fields = result["additionalModelRequestFields"] + assert "anthropic_beta" in additional_fields + assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"] + + else: + config = AmazonAnthropicClaudeConfig() + result = config.transform_request( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers=headers, + ) + + # Verify beta header is in top-level request + assert "anthropic_beta" in result + assert "context-1m-2025-08-07" in result["anthropic_beta"] + + @pytest.mark.parametrize( + "model_prefix", + [ + "bedrock/invoke/", + "bedrock/converse/", + ], + ) + def test_1m_context_with_multiple_beta_headers(self, model_prefix): + """ + Test that 1M context header works alongside other beta headers. + + Ensures that multiple anthropic-beta values (comma-separated) are all + correctly passed through. + """ + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + + # Multiple beta headers including 1M context + headers = { + "anthropic-beta": "context-1m-2025-08-07,computer-use-2024-10-22" + } + messages = [{"role": "user", "content": "Test"}] + + if "converse" in model_prefix: + config = AmazonConverseConfig() + result = config._transform_request_helper( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + system_content_blocks=[], + optional_params={}, + messages=messages, + headers=headers, + ) + + additional_fields = result["additionalModelRequestFields"] + beta_headers = additional_fields["anthropic_beta"] + + else: + config = AmazonAnthropicClaudeConfig() + result = config.transform_request( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers=headers, + ) + + beta_headers = result["anthropic_beta"] + + # Verify both headers are present + assert "context-1m-2025-08-07" in beta_headers + assert "computer-use-2024-10-22" in beta_headers + + +class TestBedrockAnthropicCombinedRegressions: + """ + Tests that combine multiple features to ensure they work together. + """ + + @pytest.mark.parametrize( + "model_prefix", + [ + "bedrock/invoke/", + "bedrock/converse/", + ], + ) + def test_1m_context_with_prompt_caching(self, model_prefix): + """ + Test that 1M context and prompt caching work together. + + This is a real-world scenario where a user might want to use both features + simultaneously. + """ + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + + headers = {"anthropic-beta": "context-1m-2025-08-07"} + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": LARGE_DOCUMENT_FOR_CACHING, + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "text", + "text": "Summarize this document.", + }, + ], + } + ] + + if "converse" in model_prefix: + config = AmazonConverseConfig() + result = config._transform_request_helper( + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + system_content_blocks=[], + optional_params={}, + messages=messages, + headers=headers, + ) + + # Should have 1M context header + additional_fields = result["additionalModelRequestFields"] + assert "anthropic_beta" in additional_fields + assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"] + + # Should NOT have prompt-caching header + assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"] + + else: + config = AmazonAnthropicClaudeConfig() + result = config.transform_request( + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers=headers, + ) + + # Should have 1M context header + assert "anthropic_beta" in result + assert "context-1m-2025-08-07" in result["anthropic_beta"] + + # Should NOT have prompt-caching header + assert "prompt-caching-2024-07-31" not in result["anthropic_beta"] + + # Should have cache_control in messages + user_msg = result["messages"][0] + has_cache_control = any( + isinstance(c, dict) and "cache_control" in c + for c in user_msg["content"] + ) + assert has_cache_control