diff --git a/litellm/__init__.py b/litellm/__init__.py index 71857877e53..dbf83b1173e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1755,6 +1755,9 @@ if TYPE_CHECKING: from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig, ) + from .llms.bedrock.chat.openai_chat_completions_transformation import ( + AmazonBedrockOpenAIChatCompletionsConfig as AmazonBedrockOpenAIChatCompletionsConfig, + ) from .llms.bedrock.image_generation.amazon_stability1_transformation import ( AmazonStabilityConfig as AmazonStabilityConfig, ) @@ -1861,6 +1864,9 @@ if TYPE_CHECKING: from .llms.bedrock_mantle.responses.transformation import ( BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig, ) + from .llms.bedrock.responses.transformation import ( + AmazonBedrockResponsesAPIConfig as AmazonBedrockResponsesAPIConfig, + ) from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 4c478b51ed1..6c9985a1d40 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -204,6 +204,7 @@ LLM_CONFIG_NAMES: Final = ( "AmazonTwelveLabsPegasusConfig", "AmazonInvokeConfig", "AmazonBedrockOpenAIConfig", + "AmazonBedrockOpenAIChatCompletionsConfig", "AmazonStabilityConfig", "AmazonStability3Config", "AmazonNovaCanvasConfig", @@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = ( "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", "BedrockMantleResponsesAPIConfig", + "AmazonBedrockResponsesAPIConfig", "GoogleAIStudioInteractionsConfig", "VertexAIInteractionsConfig", "OpenAIOSeriesConfig", @@ -848,6 +850,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation", "AmazonBedrockOpenAIConfig", ), + "AmazonBedrockOpenAIChatCompletionsConfig": ( + ".llms.bedrock.chat.openai_chat_completions_transformation", + "AmazonBedrockOpenAIChatCompletionsConfig", + ), "AmazonStabilityConfig": ( ".llms.bedrock.image_generation.amazon_stability1_transformation", "AmazonStabilityConfig", @@ -991,6 +997,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.bedrock_mantle.responses.transformation", "BedrockMantleResponsesAPIConfig", ), + "AmazonBedrockResponsesAPIConfig": ( + ".llms.bedrock.responses.transformation", + "AmazonBedrockResponsesAPIConfig", + ), "GoogleAIStudioInteractionsConfig": ( ".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig", diff --git a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py new file mode 100644 index 00000000000..279bafe1081 --- /dev/null +++ b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py @@ -0,0 +1,113 @@ +from typing import Final + +import httpx + +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.request_metadata import ( + bedrock_request_metadata_headers, + merge_bedrock_invoke_headers, +) +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.llms.openai import AllMessageValues + + +class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: provider interface + OpenAIGPTConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> str: + return "bedrock" + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + stream: bool | None = None, + ) -> str: + aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model=model) + endpoint_url: Final = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=optional_params.get("aws_bedrock_runtime_endpoint"), + aws_region_name=aws_region_name, + )[0] + return f"{endpoint_url.rstrip('/')}/openai/v1/chat/completions" + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + headers: dict, # mutable-ok: provider interface + ) -> dict: # mutable-ok: provider interface + inference_params: Final = { # mutable-ok: provider interface + key: value for key, value in optional_params.items() if key not in self.aws_authentication_params + } + return super().transform_request( + model=model.removeprefix("bedrock/"), + messages=messages, + optional_params=inference_params, + litellm_params=litellm_params, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, # mutable-ok: provider interface + model: str, + messages: list[AllMessageValues], # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: # mutable-ok: provider interface + owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params) + return merge_bedrock_invoke_headers( + { + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "Content-Type": next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ), + }, + (), + metadata_headers, + owned_names, + ) + + def sign_request( + self, + headers: dict, # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + request_data: dict, # mutable-ok: provider interface + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface + ) -> BedrockError: + return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index cb2c70e74c8..d0a64662d8e 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -34,6 +34,43 @@ if TYPE_CHECKING: _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" +_BEDROCK_ROUTE_PREFIXES: Final[tuple[str, ...]] = ( + "invoke/", + "claude_platform/", + "converse_like/", + "converse/", + "agent/", + "agentcore/", + "async_invoke/", + "openai/", + "mantle/", +) +_BEDROCK_ROUTE_TYPES: Final[ + tuple[ + Literal[ + "invoke", + "claude_platform", + "converse_like", + "converse", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", + ], + ..., + ] +] = ( + "invoke", + "claude_platform", + "converse_like", + "converse", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", +) def error_response_text(response: httpx.Response) -> str: @@ -1089,6 +1126,10 @@ class BedrockModelInfo(BaseLLMModelInfo): """Wrapper for standalone function. See get_bedrock_base_model().""" return get_bedrock_base_model(model) + @staticmethod + def has_explicit_route(model: str) -> bool: + return any(BedrockModelInfo._model_has_route_prefix(model, prefix) for prefix in _BEDROCK_ROUTE_PREFIXES) + @staticmethod def _supported_cross_region_inference_region() -> list[str]: """Wrapper for standalone function. See get_bedrock_cross_region_inference_regions().""" @@ -1111,36 +1152,11 @@ class BedrockModelInfo(BaseLLMModelInfo): """ Get the bedrock route for the given model. """ - route_mappings: dict[ - str, - Literal[ - "invoke", - "claude_platform", - "converse_like", - "converse", - "agent", - "agentcore", - "async_invoke", - "openai", - "mantle", - ], - ] = { - "invoke/": "invoke", - "claude_platform/": "claude_platform", - "converse_like/": "converse_like", - "converse/": "converse", - "agent/": "agent", - "agentcore/": "agentcore", - "async_invoke/": "async_invoke", - "openai/": "openai", - "mantle/": "mantle", - } - # Check explicit routes first. Match each prefix only as a leading path # segment so the `bedrock_mantle/` provider prefix is never mistaken for # the `mantle/` invoke route (which would mangle # `bedrock_mantle/openai.gpt-5.5` into `bedrock_openai.gpt-5.5`). - for prefix, route_type in route_mappings.items(): + for prefix, route_type in zip(_BEDROCK_ROUTE_PREFIXES, _BEDROCK_ROUTE_TYPES, strict=True): if BedrockModelInfo._model_has_route_prefix(model, prefix): return route_type @@ -1292,6 +1308,17 @@ class BedrockModelInfo(BaseLLMModelInfo): return AmazonMantleMessagesConfig() + from litellm.utils import model_supports_native_endpoint + + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/messages", model, litellm.LlmProviders.BEDROCK + ): + from litellm.llms.bedrock.messages.native_transformation import ( + AmazonBedrockNativeMessagesConfig, + ) + + return AmazonBedrockNativeMessagesConfig() + ######################################################### # This goes through litellm.AmazonAnthropicClaude3MessagesConfig() # Since bedrock Invoke supports Native Anthropic Messages API @@ -1321,6 +1348,13 @@ def get_bedrock_chat_config(model: str): bedrock_invoke_provider: Final = BaseAWSLLM.get_bedrock_invoke_provider(model=model) base_model: Final = BedrockModelInfo.get_base_model(model) + from litellm.utils import model_supports_native_endpoint + + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/chat/completions", model, litellm.LlmProviders.BEDROCK + ): + return litellm.AmazonBedrockOpenAIChatCompletionsConfig() + # Handle explicit routes first if bedrock_route == "claude_platform": return litellm.BedrockClaudePlatformConfig() diff --git a/litellm/llms/bedrock/messages/native_transformation.py b/litellm/llms/bedrock/messages/native_transformation.py new file mode 100644 index 00000000000..ca481851ced --- /dev/null +++ b/litellm/llms/bedrock/messages/native_transformation.py @@ -0,0 +1,136 @@ +from typing import Final, Protocol + +import httpx + +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + AnthropicMessagesConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token +from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url +from litellm.types.router import GenericLiteLLMParams + + +class MessagesURLBuilder(Protocol): + def __call__(self, api_base: str | None, endpoint: str | None, region: str) -> str: ... + + +class BearerTokenResolver(Protocol): + def __call__(self, api_key: str | None) -> str | None: ... + + +class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig): + def __init__( + self, + custom_llm_provider: str = "bedrock", + url_builder: MessagesURLBuilder | None = None, + bearer_token_resolver: BearerTokenResolver = bedrock_bearer_token, + **kwargs: object, # kwargs-ok: provider interface + ) -> None: + BaseAWSLLM.__init__(self, **kwargs) + self._custom_llm_provider = custom_llm_provider + self._url_builder = url_builder + self._bearer_token_resolver = bearer_token_resolver + + @property + def custom_llm_provider(self) -> str: + return self._custom_llm_provider + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + stream: bool | None = None, + ) -> str: + region: Final = self._get_aws_region_name(optional_params=optional_params, model=model) + endpoint: Final = optional_params.get("aws_bedrock_runtime_endpoint") + if self._url_builder is not None: + return self._url_builder(api_base, endpoint, region) + runtime_endpoint: Final = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=endpoint, + aws_region_name=region, + )[0] + return f"{runtime_endpoint.rstrip('/')}/anthropic/v1/messages" + + def validate_anthropic_messages_environment( + self, + headers: dict, # mutable-ok: provider interface + model: str, + messages: list[dict], # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict, str | None]: # mutable-ok: provider interface + content_type: Final = next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ) + return ( + { # mutable-ok: provider interface + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "anthropic-version": headers.get("anthropic-version", DEFAULT_ANTHROPIC_API_VERSION), + "Content-Type": content_type, + }, + api_base, + ) + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict], # mutable-ok: provider interface + anthropic_messages_optional_request_params: dict, # mutable-ok: provider interface + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: provider interface + ) -> dict: # mutable-ok: provider interface + return { # mutable-ok: provider interface + **anthropic_messages_optional_request_params, + "model": model, + "messages": messages, + } + + def sign_request( + self, + headers: dict, # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + request_data: dict, # mutable-ok: provider interface + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=self._bearer_token_resolver(api_key), + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message, headers=headers) + + +def mantle_native_messages_config() -> AmazonBedrockNativeMessagesConfig: + from litellm.llms.bedrock_mantle.common_utils import resolve_mantle_bearer_token + + return AmazonBedrockNativeMessagesConfig( + custom_llm_provider="bedrock_mantle", + url_builder=build_mantle_messages_url, + bearer_token_resolver=resolve_mantle_bearer_token, + ) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py new file mode 100644 index 00000000000..8fc065fcde7 --- /dev/null +++ b/litellm/llms/bedrock/responses/transformation.py @@ -0,0 +1,91 @@ +from typing import Final + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ResponseInputParam +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: provider interface + OpenAIResponsesAPIConfig.__init__(self) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK + + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict, # mutable-ok: provider interface + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: provider interface + ) -> dict: # mutable-ok: provider interface + request_params: Final = { # mutable-ok: provider interface + key: value + for key, value in response_api_optional_request_params.items() + if key not in self.aws_authentication_params + } + return { # mutable-ok: provider interface + **request_params, + "model": model, + "input": input, + } + + def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: # mutable-ok: provider interface + region: Final = self._get_aws_region_name(optional_params=litellm_params, model="") + endpoint_url: Final = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=litellm_params.get("aws_bedrock_runtime_endpoint"), + aws_region_name=region, + )[0] + return f"{endpoint_url.rstrip('/')}/openai/v1/responses" + + def validate_environment( + self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict: # mutable-ok: provider interface + return { + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "Content-Type": next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ), + } # mutable-ok: provider interface + + def sign_request( + self, + headers: dict, # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + request_data: dict, # mutable-ok: provider interface + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/main.py b/litellm/main.py index 22d59520f74..227b4560a79 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4062,6 +4062,10 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + from litellm.llms.bedrock.chat.openai_chat_completions_transformation import ( + AmazonBedrockOpenAIChatCompletionsConfig, + ) + if "aws_bedrock_client" in optional_params: verbose_logger.warning( "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." @@ -4080,7 +4084,27 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) - if bedrock_route == "claude_platform": + if isinstance(provider_config, AmazonBedrockOpenAIChatCompletionsConfig): + return base_llm_http_handler.completion( + model=model.removeprefix("bedrock/"), + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + elif bedrock_route == "claude_platform": provider_config = ProviderConfigManager.get_provider_chat_config( model=model, provider=LlmProviders.BEDROCK, diff --git a/litellm/utils.py b/litellm/utils.py index 2c9200fbad7..1b63839d874 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4550,7 +4550,17 @@ def get_optional_params( BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo") bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) bedrock_base_model: Final = BedrockModelInfo.get_base_model(model) - if bedrock_route == "converse" or bedrock_route == "converse_like": + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/chat/completions", model, LlmProviders.BEDROCK + ): + native_provider_config: Final = provider_config or litellm.AmazonBedrockOpenAIChatCompletionsConfig() + optional_params = native_provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + ) + elif bedrock_route == "converse" or bedrock_route == "converse_like": optional_params = litellm.AmazonConverseConfig().map_openai_params( model=model, non_default_params=non_default_params, @@ -8148,7 +8158,7 @@ def _get_model_cost_entry_for_provider_config( model: str, provider: LlmProviders, ) -> dict[str, Any]: - candidate_keys: Final = (model, f"{provider.value}/{model}") + candidate_keys: Final = (f"{provider.value}/{model}", model) for model_key in candidate_keys: model_info = litellm.model_cost.get(model_key) if model_info is not None: @@ -8162,6 +8172,22 @@ def _get_model_cost_entry_for_provider_config( return {} +def model_supports_native_endpoint(endpoint: str, model: str, provider: LlmProviders) -> bool: + bedrock_model_info: Final = litellm.BedrockModelInfo + base_model: Final = bedrock_model_info.get_base_model(model) if provider is LlmProviders.BEDROCK else model + candidate_models: Final = (model, base_model) if base_model != model else (model,) + return any( + isinstance( + supported_endpoints := _get_model_cost_entry_for_provider_config(candidate_model, provider).get( + "supported_endpoints" + ), + (list, tuple), + ) + and endpoint in supported_endpoints + for candidate_model in candidate_models + ) + + class ProviderConfigManager: # Dictionary mapping for O(1) provider lookup # Stores tuples of (factory_function, needs_model_parameter) @@ -8620,6 +8646,22 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if ( + provider is LlmProviders.BEDROCK + and not litellm.BedrockModelInfo.has_explicit_route(model) + and model_supports_native_endpoint("/v1/messages", model, LlmProviders.BEDROCK) + ): + from litellm.llms.bedrock.messages.native_transformation import ( + AmazonBedrockNativeMessagesConfig, + ) + + return AmazonBedrockNativeMessagesConfig() + if provider is LlmProviders.BEDROCK_MANTLE and model_supports_native_endpoint( + "/v1/messages", model, LlmProviders.BEDROCK_MANTLE + ): + from litellm.llms.bedrock.messages.native_transformation import mantle_native_messages_config + + return mantle_native_messages_config() return ProviderConfigManager._get_provider_anthropic_messages_config_cached(model=model, provider=provider) @staticmethod @@ -8902,6 +8944,12 @@ class ProviderConfigManager: return litellm.BedrockMantleResponsesAPIConfig( use_openai_path=mantle_base_segment(model, litellm.model_cost) == "openai/v1" ) + elif litellm.LlmProviders.BEDROCK == provider: + if model is None or not model_supports_native_endpoint("/v1/responses", model, LlmProviders.BEDROCK): + return None + from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig + + return AmazonBedrockResponsesAPIConfig() return None @staticmethod diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py new file mode 100644 index 00000000000..138f861eeb7 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -0,0 +1,410 @@ +import json +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm.llms.bedrock.chat.openai_chat_completions_transformation import ( + AmazonBedrockOpenAIChatCompletionsConfig, +) +from litellm.llms.bedrock.common_utils import get_bedrock_chat_config +from litellm.llms.bedrock.messages.native_transformation import ( + AmazonBedrockNativeMessagesConfig, + mantle_native_messages_config, +) +from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, model_supports_native_endpoint + + +def _chat_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ # mutable-ok: provider interface + "id": "chat-test", + "object": "chat.completion", + "created": 1, + "model": "native-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], # mutable-ok: provider interface + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, # mutable-ok: provider interface + }, + request=httpx.Request("POST", url), + ) + + +def _messages_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ # mutable-ok: provider interface + "id": "message-test", + "type": "message", + "role": "assistant", + "model": "native-model", + "content": [{"type": "text", "text": "ok"}], # mutable-ok: provider interface + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, # mutable-ok: provider interface + }, + request=httpx.Request("POST", url), + ) + + +def _responses_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ # mutable-ok: provider interface + "id": "response-test", + "object": "response", + "created_at": 1, + "model": "native-model", + "output": [], # mutable-ok: provider interface + "status": "completed", + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, # mutable-ok: provider interface + }, + request=httpx.Request("POST", url), + ) + + +def test_native_endpoint_gate_reads_provider_model_cost(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert model_supports_native_endpoint( + "/v1/chat/completions", "native-model", LlmProviders.BEDROCK + ) + assert not model_supports_native_endpoint("/v1/messages", "native-model", LlmProviders.BEDROCK) + + +def test_native_endpoint_gate_uses_bedrock_base_model(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-3-7-sonnet", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + + assert model_supports_native_endpoint( + "/v1/messages", "us.anthropic.claude-3-7-sonnet", LlmProviders.BEDROCK + ) + + +def test_bedrock_chat_route_prefers_explicit_converse(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert isinstance(get_bedrock_chat_config("native-model"), AmazonBedrockOpenAIChatCompletionsConfig) + assert type(get_bedrock_chat_config("converse/native-model")).__name__ == "AmazonConverseConfig" + + +def test_bedrock_converse_catalog_model_prefers_declared_native_chat(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert isinstance( + get_bedrock_chat_config("anthropic.claude-sonnet-4-6"), + AmazonBedrockOpenAIChatCompletionsConfig, + ) + assert type(get_bedrock_chat_config("converse/anthropic.claude-sonnet-4-6")).__name__ == "AmazonConverseConfig" + + +def test_bedrock_messages_route_prefers_explicit_invoke(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + + assert isinstance( + ProviderConfigManager.get_provider_anthropic_messages_config( + model="anthropic.claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ), + AmazonBedrockNativeMessagesConfig, + ) + assert type( + ProviderConfigManager.get_provider_anthropic_messages_config( + model="invoke/anthropic.claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + ).__name__ == "AmazonAnthropicClaudeMessagesConfig" + + +def test_native_chat_url_and_bearer_auth(monkeypatch): + config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture + + assert config.get_complete_url( + None, None, "native-model", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface + ) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") + validated_headers = config.validate_environment( + {"content-type": "application/json"}, # mutable-ok: provider interface + "native-model", + [], # mutable-ok: provider interface + {}, # mutable-ok: provider interface + {}, # mutable-ok: provider interface + ) + headers, body = config.sign_request( + validated_headers, + {"aws_region_name": "us-west-2"}, # mutable-ok: provider interface + {"model": "native-model"}, # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", + ) + assert headers["Authorization"] == "Bearer token" + assert sum(key.lower() == "content-type" for key in headers) == 1 + assert body is not None + + +def test_native_chat_sigv4_auth(monkeypatch): + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture + headers, _ = config.sign_request( + {}, # mutable-ok: provider interface + { # mutable-ok: provider interface + "aws_region_name": "us-west-2", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing", + }, + {"model": "native-model"}, # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", + ) + + assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") + + +def test_native_messages_preserves_body_and_urls(monkeypatch): + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") + config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture + request = { # mutable-ok: provider interface # rebind-ok: test capture + "model": "claude-native", + "messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface + "stream": True, + "cache_control": {"ttl": "5m"}, # mutable-ok: provider interface + } + body = config.transform_anthropic_messages_request( # rebind-ok: test capture + model=request["model"], + messages=request["messages"], + anthropic_messages_optional_request_params={ # mutable-ok: provider interface + "stream": request["stream"], + "cache_control": request["cache_control"], + }, + litellm_params={}, # mutable-ok: provider interface + headers={}, # mutable-ok: provider interface + ) + + assert body == request + assert "anthropic_version" not in body + assert config.get_complete_url(None, None, "claude-native", {"aws_region_name": "us-west-2"}, {}) == ( # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages" + ) + assert mantle_native_messages_config().get_complete_url( + None, None, "claude-native", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface + ) == "https://bedrock-mantle.us-west-2.api.aws/anthropic/v1/messages" + headers, signed_body = config.sign_request( + {}, # mutable-ok: provider interface + {"aws_region_name": "us-west-2"}, # mutable-ok: provider interface + body, # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages", + ) + assert headers["Authorization"] == "Bearer token" + assert sum(key.lower() == "content-type" for key in headers) == 1 + assert signed_body is not None + + +def test_native_messages_deduplicates_content_type_header(monkeypatch): + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") + config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture + headers, _ = config.validate_anthropic_messages_environment( + headers={"content-type": "application/json"}, # mutable-ok: provider interface + model="native-model", + messages=[], # mutable-ok: provider interface + optional_params={}, # mutable-ok: provider interface + litellm_params={}, # mutable-ok: provider interface + ) + + assert sum(key.lower() == "content-type" for key in headers) == 1 + assert headers["Content-Type"] == "application/json" + + +def test_native_messages_selection_is_not_cached(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + assert isinstance( + ProviderConfigManager.get_provider_anthropic_messages_config( + "native-model", LlmProviders.BEDROCK + ), + AmazonBedrockNativeMessagesConfig, + ) + + monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface + assert type( + ProviderConfigManager.get_provider_anthropic_messages_config( + "claude-native", LlmProviders.BEDROCK + ) + ).__name__ == "AmazonAnthropicClaudeMessagesConfig" + + +def test_native_responses_selection_and_url(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/responses"]}, # mutable-ok: provider interface + ) + config = ProviderConfigManager.get_provider_responses_api_config( # rebind-ok: test capture + LlmProviders.BEDROCK, "native-model" + ) + + assert isinstance(config, AmazonBedrockResponsesAPIConfig) + assert config.get_complete_url(None, {"aws_region_name": "us-west-2"}).endswith( # mutable-ok: provider interface + "/openai/v1/responses" + ) + validated_headers = config.validate_environment( + {"content-type": "application/json"}, # mutable-ok: provider interface + "native-model", + {}, # mutable-ok: provider interface + ) + assert sum(key.lower() == "content-type" for key in validated_headers) == 1 + + monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface + assert ProviderConfigManager.get_provider_responses_api_config( + LlmProviders.BEDROCK, "native-model" + ) is None + + +def test_completion_forwards_native_chat_body(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _chat_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.completion( + model="bedrock/native-model", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests == [ # mutable-ok: provider interface + ( + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", + {"model": "native-model", "messages": [{"role": "user", "content": "hello"}]}, # mutable-ok: provider interface + ) + ] + + +def test_completion_forwards_declared_native_chat_for_converse_catalog_model(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _chat_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.completion( + model="bedrock/anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests[0][0].endswith("/openai/v1/chat/completions") + + +@pytest.mark.asyncio +async def test_anthropic_messages_forwards_native_body(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + async def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _messages_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=post + ): + await litellm.anthropic_messages( + model="bedrock/native-model", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + max_tokens=10, + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests == [ # mutable-ok: provider interface + ( + "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages", + { # mutable-ok: provider interface + "max_tokens": 10, + "stream": False, + "model": "native-model", + "messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface + }, + ) + ] + + +def test_responses_forwards_native_body(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/responses"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _responses_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.responses( + model="bedrock/native-model", + input="hello", + instructions="be brief", + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests == [ # mutable-ok: provider interface + ( + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses", + { # mutable-ok: provider interface + "model": "native-model", + "input": "hello", + "instructions": "be brief", + }, + ) + ]