From d3be5c3ab72dc5a65e5353c3c6dfc977fba90f96 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:39:34 +0000 Subject: [PATCH] test(integration): bedrock_converse-route basic translation cases on messages, chat completions and responses (#44646) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/proxy_config.yaml | 36 ++ .../bases/bedrock_converse.py | 346 ++++++++++++ ...chat_completions_basic_bedrock_converse.py | 31 ++ .../messages/bases/bedrock_converse.py | 274 ++++++++++ .../test_messages_basic_bedrock_converse.py | 29 + .../responses/bases/bedrock_converse.py | 502 ++++++++++++++++++ .../test_responses_basic_bedrock_converse.py | 31 ++ 7 files changed, 1249 insertions(+) create mode 100644 tests/integration/translation/chat_completions/bases/bedrock_converse.py create mode 100644 tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py create mode 100644 tests/integration/translation/messages/bases/bedrock_converse.py create mode 100644 tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py create mode 100644 tests/integration/translation/responses/bases/bedrock_converse.py create mode 100644 tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index d0298da0dd9..a691b155bb2 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -29,6 +29,42 @@ model_list: model: anthropic/claude-sonnet-4-6 api_base: http://127.0.0.1:8191 api_key: synthetic-anthropic-key + - model_name: bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0 + litellm_params: + model: bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 + - model_name: bedrock/converse/us.anthropic.claude-sonnet-4-6 + litellm_params: + model: bedrock/converse/us.anthropic.claude-sonnet-4-6 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 + - model_name: bedrock/converse/us.anthropic.claude-sonnet-5 + litellm_params: + model: bedrock/converse/us.anthropic.claude-sonnet-5 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 + - model_name: bedrock/converse/us.anthropic.claude-opus-4-8 + litellm_params: + model: bedrock/converse/us.anthropic.claude-opus-4-8 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 + - model_name: bedrock/converse/us.anthropic.claude-sonnet-5-5 + litellm_params: + model: bedrock/converse/us.anthropic.claude-sonnet-5-5 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 + - model_name: bedrock/converse/us.anthropic.claude-opus-5-5 + litellm_params: + model: bedrock/converse/us.anthropic.claude-opus-5-5 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/integration/translation/chat_completions/bases/bedrock_converse.py b/tests/integration/translation/chat_completions/bases/bedrock_converse.py new file mode 100644 index 00000000000..59c677540b6 --- /dev/null +++ b/tests/integration/translation/chat_completions/bases/bedrock_converse.py @@ -0,0 +1,346 @@ +from typing import Final +from unittest.mock import ANY + +from integration.translation.case import TranslationTestCase + +CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 771}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 17, + "outputTokens": 5, + "serverToolUsage": {}, + "totalTokens": 22, + }, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "object": "chat.completion", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 17, + "total_tokens": 22, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 5}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 17, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + +CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-6", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-4-6/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 997}, + "output": {"message": {"content": [{"text": "Hello!"}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 18, + "outputTokens": 5, + "serverToolUsage": {}, + "totalTokens": 23, + }, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-6", + "object": "chat.completion", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 18, + "total_tokens": 23, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 5}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 18, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + +CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-5", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 1201}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 21, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 27, + }, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5", + "object": "chat.completion", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "usage": { + "completion_tokens": 6, + "prompt_tokens": 21, + "total_tokens": 27, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 21, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + +CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-opus-4-8", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-opus-4-8/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 827}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 21, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 27, + }, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "model": "bedrock/converse/us.anthropic.claude-opus-4-8", + "object": "chat.completion", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "usage": { + "completion_tokens": 6, + "prompt_tokens": 21, + "total_tokens": 27, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 21, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 892}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 23, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 29, + }, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "object": "chat.completion", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "usage": { + "completion_tokens": 6, + "prompt_tokens": 23, + "total_tokens": 29, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 23, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 10460}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 23, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 29, + }, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "object": "chat.completion", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "usage": { + "completion_tokens": 6, + "prompt_tokens": 23, + "total_tokens": 29, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 23, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py new file mode 100644 index 00000000000..f12143b5752 --- /dev/null +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py @@ -0,0 +1,31 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.bases.bedrock_converse import ( + CLAUDE_HAIKU_4_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, + CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + CLAUDE_HAIKU_4_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_OPUS_5_5_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_chat_completions_basic_bedrock_converse( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/messages/bases/bedrock_converse.py b/tests/integration/translation/messages/bases/bedrock_converse.py new file mode 100644 index 00000000000..fbb69d3a93a --- /dev/null +++ b/tests/integration/translation/messages/bases/bedrock_converse.py @@ -0,0 +1,274 @@ +from typing import Final +from unittest.mock import ANY + +from integration.translation.case import TranslationTestCase + +CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 771}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 17, + "outputTokens": 5, + "serverToolUsage": {}, + "totalTokens": 22, + }, + }, + expected_litellm_response={ + "id": ANY, + "type": "message", + "role": "assistant", + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "stop_sequence": None, + "usage": {"input_tokens": 17, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-6", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-4-6/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 997}, + "output": {"message": {"content": [{"text": "Hello!"}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 18, + "outputTokens": 5, + "serverToolUsage": {}, + "totalTokens": 23, + }, + }, + expected_litellm_response={ + "id": ANY, + "type": "message", + "role": "assistant", + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-6", + "stop_sequence": None, + "usage": {"input_tokens": 18, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-5", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 1201}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 21, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 27, + }, + }, + expected_litellm_response={ + "id": ANY, + "type": "message", + "role": "assistant", + "model": "bedrock/converse/us.anthropic.claude-sonnet-5", + "stop_sequence": None, + "usage": {"input_tokens": 21, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-opus-4-8", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-opus-4-8/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 827}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 21, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 27, + }, + }, + expected_litellm_response={ + "id": ANY, + "type": "message", + "role": "assistant", + "model": "bedrock/converse/us.anthropic.claude-opus-4-8", + "stop_sequence": None, + "usage": {"input_tokens": 21, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 892}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 23, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 29, + }, + }, + expected_litellm_response={ + "id": ANY, + "type": "message", + "role": "assistant", + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "stop_sequence": None, + "usage": {"input_tokens": 23, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 10460}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 23, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 29, + }, + }, + expected_litellm_response={ + "id": ANY, + "type": "message", + "role": "assistant", + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "stop_sequence": None, + "usage": {"input_tokens": 23, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py new file mode 100644 index 00000000000..1b636b829ab --- /dev/null +++ b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py @@ -0,0 +1,29 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.bedrock_converse import ( + CLAUDE_HAIKU_4_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, + CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + CLAUDE_HAIKU_4_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_OPUS_5_5_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_messages_basic_bedrock_converse(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/responses/bases/bedrock_converse.py b/tests/integration/translation/responses/bases/bedrock_converse.py new file mode 100644 index 00000000000..313d962dc42 --- /dev/null +++ b/tests/integration/translation/responses/bases/bedrock_converse.py @@ -0,0 +1,502 @@ +from typing import Final +from unittest.mock import ANY + +from integration.translation.case import TranslationTestCase + +CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 771}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 17, + "outputTokens": 5, + "serverToolUsage": {}, + "totalTokens": 22, + }, + }, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 17, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 5, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 5}, + "total_tokens": 22, + "cost": None, + }, + "user": None, + "store": None, + }, +) + +CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-6", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-4-6/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 997}, + "output": {"message": {"content": [{"text": "Hello!"}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 18, + "outputTokens": 5, + "serverToolUsage": {}, + "totalTokens": 23, + }, + }, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-6", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello!", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 18, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 18, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 5, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 5}, + "total_tokens": 23, + "cost": None, + }, + "user": None, + "store": None, + }, +) + +CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-5", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 1201}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 21, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 27, + }, + }, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 21, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 21, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, + "total_tokens": 27, + "cost": None, + }, + "user": None, + "store": None, + }, +) + +CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-opus-4-8", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-opus-4-8/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 827}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 21, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 27, + }, + }, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "bedrock/converse/us.anthropic.claude-opus-4-8", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 21, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 21, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, + "total_tokens": 27, + "cost": None, + }, + "user": None, + "store": None, + }, +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 892}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 23, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 29, + }, + }, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 23, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 23, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, + "total_tokens": 29, + "cost": None, + }, + "user": None, + "store": None, + }, +) + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], + "inferenceConfig": {"maxTokens": 64}, + "system": [{"text": "You are a terse assistant."}], + }, + mock_provider_response={ + "metrics": {"latencyMs": 10460}, + "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 23, + "outputTokens": 6, + "serverToolUsage": {}, + "totalTokens": 29, + }, + }, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 23, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 23, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, + "total_tokens": 29, + "cost": None, + }, + "user": None, + "store": None, + }, +) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py new file mode 100644 index 00000000000..52bf9729180 --- /dev/null +++ b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py @@ -0,0 +1,31 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.bedrock_converse import ( + CLAUDE_HAIKU_4_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, + CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + CLAUDE_HAIKU_4_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_OPUS_5_5_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_responses_basic_bedrock_converse( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + run(case, gateway, provider)