From b137257b417c926fbf1ecb2889518e8061630bd9 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 03:02:03 +0000 Subject: [PATCH 1/9] feat(bedrock): forward declared native endpoints without translation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 6 + litellm/_lazy_imports_registry.py | 10 + .../messages/handler.py | 26 +- .../messages/native_utils.py | 51 +++ .../openai_chat_completions_transformation.py | 104 ++++++ litellm/llms/bedrock/common_utils.py | 15 + .../bedrock/messages/native_transformation.py | 131 +++++++ .../llms/bedrock/responses/transformation.py | 80 +++++ litellm/main.py | 26 +- litellm/responses/main.py | 17 +- litellm/utils.py | 37 ++ .../test_native_endpoint_forwarding.py | 333 ++++++++++++++++++ 12 files changed, 830 insertions(+), 6 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py create mode 100644 litellm/llms/bedrock/chat/openai_chat_completions_transformation.py create mode 100644 litellm/llms/bedrock/messages/native_transformation.py create mode 100644 litellm/llms/bedrock/responses/transformation.py create mode 100644 tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py diff --git a/litellm/__init__.py b/litellm/__init__.py index ccfbf80369f..7423c4767c9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1737,6 +1737,9 @@ if TYPE_CHECKING: from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig, ) + from .llms.bedrock.chat.openai_chat_completions_transformation import ( + AmazonBedrockOpenAIChatCompletionsConfig as AmazonBedrockOpenAIChatCompletionsConfig, + ) from .llms.bedrock.image_generation.amazon_stability1_transformation import ( AmazonStabilityConfig as AmazonStabilityConfig, ) @@ -1840,6 +1843,9 @@ if TYPE_CHECKING: from .llms.bedrock_mantle.responses.transformation import ( BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig, ) + from .llms.bedrock.responses.transformation import ( + AmazonBedrockResponsesAPIConfig as AmazonBedrockResponsesAPIConfig, + ) from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index dc323c8cc15..149da725c5e 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -204,6 +204,7 @@ LLM_CONFIG_NAMES: Final = ( "AmazonTwelveLabsPegasusConfig", "AmazonInvokeConfig", "AmazonBedrockOpenAIConfig", + "AmazonBedrockOpenAIChatCompletionsConfig", "AmazonStabilityConfig", "AmazonStability3Config", "AmazonNovaCanvasConfig", @@ -243,6 +244,7 @@ LLM_CONFIG_NAMES: Final = ( "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", "BedrockMantleResponsesAPIConfig", + "AmazonBedrockResponsesAPIConfig", "GoogleAIStudioInteractionsConfig", "VertexAIInteractionsConfig", "OpenAIOSeriesConfig", @@ -847,6 +849,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation", "AmazonBedrockOpenAIConfig", ), + "AmazonBedrockOpenAIChatCompletionsConfig": ( + ".llms.bedrock.chat.openai_chat_completions_transformation", + "AmazonBedrockOpenAIChatCompletionsConfig", + ), "AmazonStabilityConfig": ( ".llms.bedrock.image_generation.amazon_stability1_transformation", "AmazonStabilityConfig", @@ -986,6 +992,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.bedrock_mantle.responses.transformation", "BedrockMantleResponsesAPIConfig", ), + "AmazonBedrockResponsesAPIConfig": ( + ".llms.bedrock.responses.transformation", + "AmazonBedrockResponsesAPIConfig", + ), "GoogleAIStudioInteractionsConfig": ( ".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig", diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 9d1e921cce4..2bf61bb65fb 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -31,13 +31,14 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import CallTypes +from litellm.types.utils import CallTypes, LlmProviders from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from ..utils import is_reasoning_auto_summary_enabled from .interceptors import get_messages_interceptors +from .native_utils import get_native_messages_request_params from .utils import AnthropicMessagesRequestUtils, mock_response # Providers that are routed directly to the OpenAI Responses API instead of @@ -566,8 +567,8 @@ def anthropic_messages_handler( model=model, provider=litellm.LlmProviders(custom_llm_provider), ) - if anthropic_messages_provider_config is None and _deployment_passes_through_anthropic_messages( - kwargs.get("model_info") + if anthropic_messages_provider_config is None and _deployment_supports_native_anthropic_messages( + kwargs.get("model_info"), model=model, custom_llm_provider=custom_llm_provider ): from litellm.llms.openai_like.messages.transformation import ( OpenAILikeAnthropicMessagesConfig, @@ -641,6 +642,11 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, ) ) + forwarded_anthropic_messages_optional_request_params: Final = get_native_messages_request_params( + optional_params=anthropic_messages_optional_request_params, + kwargs=kwargs, + preserves_request_body=getattr(anthropic_messages_provider_config, "preserves_request_body", False), + ) if is_reasoning_auto_summary_enabled(): thinking_param: Final = anthropic_messages_optional_request_params.get("thinking") if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled": @@ -653,7 +659,7 @@ def anthropic_messages_handler( model=model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params), + anthropic_messages_optional_request_params=forwarded_anthropic_messages_optional_request_params, _is_async=is_async, client=client, custom_llm_provider=custom_llm_provider, @@ -664,3 +670,15 @@ def anthropic_messages_handler( stream=stream, kwargs=kwargs, ) + + +def _deployment_supports_native_anthropic_messages( + model_info: object, model: str | None, custom_llm_provider: str | None +) -> bool: + if _deployment_passes_through_anthropic_messages(model_info): + return True + if model is None or custom_llm_provider not in tuple(provider.value for provider in LlmProviders): + return False + from litellm.utils import model_supports_native_endpoint + + return model_supports_native_endpoint("/v1/messages", model, LlmProviders(custom_llm_provider)) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py new file mode 100644 index 00000000000..7d4e1b6df6f --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py @@ -0,0 +1,51 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +_NATIVE_REQUEST_EXCLUDED_KEYS: Final = frozenset( + { + "api_base", + "api_key", + "client", + "custom_llm_provider", + "is_async", + "kwargs", + "litellm_logging_obj", + "litellm_call_id", + "model", + "messages", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region_name", + "aws_session_name", + "aws_profile_name", + "aws_role_name", + "aws_web_identity_token", + "aws_sts_endpoint", + "aws_bedrock_runtime_endpoint", + "aws_external_id", + "aws_session_tags", + } +) + + +def get_native_messages_request_params( + optional_params: Mapping[str, object], + kwargs: Mapping[str, object], + preserves_request_body: bool, +) -> Mapping[str, object]: + if not preserves_request_body: + return optional_params + return MappingProxyType( + { + **optional_params, + **MappingProxyType( + { + key: value + for key, value in kwargs.items() + if key not in _NATIVE_REQUEST_EXCLUDED_KEYS and value is not None + } + ), + } + ) diff --git a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py new file mode 100644 index 00000000000..1ddf7b5dd12 --- /dev/null +++ b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py @@ -0,0 +1,104 @@ +from typing import Final + +import httpx + +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.request_metadata import ( + bedrock_request_metadata_headers, + merge_bedrock_invoke_headers, +) +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.llms.openai import AllMessageValues + + +class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: provider interface + OpenAIGPTConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> str: + return "bedrock" + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + stream: bool | None = None, + ) -> str: + aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model=model) + endpoint_url: Final = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=optional_params.get("aws_bedrock_runtime_endpoint"), + aws_region_name=aws_region_name, + )[0] + return f"{endpoint_url.rstrip('/')}/openai/v1/chat/completions" + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + headers: dict, # mutable-ok: provider interface + ) -> dict: # mutable-ok: provider interface + inference_params: Final = { # mutable-ok: provider interface + key: value for key, value in optional_params.items() if key not in self.aws_authentication_params + } + return super().transform_request( + model=model.removeprefix("bedrock/"), + messages=messages, + optional_params=inference_params, + litellm_params=litellm_params, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, # mutable-ok: provider interface + model: str, + messages: list[AllMessageValues], # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: # mutable-ok: provider interface + owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params) + return merge_bedrock_invoke_headers( + headers, + (), + metadata_headers, + owned_names, + ) + + def sign_request( + self, + headers: dict, # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + request_data: dict, # mutable-ok: provider interface + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def get_error_class( + self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers # mutable-ok: provider interface + ) -> BedrockError: + return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index cb2c70e74c8..9f5656dd455 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -28,6 +28,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret, get_secret_str +from litellm.types.utils import LlmProviders if TYPE_CHECKING: from litellm.types.llms.openai import AllMessageValues @@ -1292,6 +1293,15 @@ class BedrockModelInfo(BaseLLMModelInfo): return AmazonMantleMessagesConfig() + from litellm.utils import model_supports_native_endpoint + + if model_supports_native_endpoint("/v1/messages", model, LlmProviders.BEDROCK): + from litellm.llms.bedrock.messages.native_transformation import ( + AmazonBedrockNativeMessagesConfig, + ) + + return AmazonBedrockNativeMessagesConfig() + ######################################################### # This goes through litellm.AmazonAnthropicClaude3MessagesConfig() # Since bedrock Invoke supports Native Anthropic Messages API @@ -1347,6 +1357,11 @@ def get_bedrock_chat_config(model: str): return AmazonMantleConfig() + from litellm.utils import model_supports_native_endpoint + + if model_supports_native_endpoint("/v1/chat/completions", model, LlmProviders.BEDROCK): + return litellm.AmazonBedrockOpenAIChatCompletionsConfig() + # Handle provider-specific configs if bedrock_invoke_provider == "amazon": return litellm.AmazonTitanConfig() diff --git a/litellm/llms/bedrock/messages/native_transformation.py b/litellm/llms/bedrock/messages/native_transformation.py new file mode 100644 index 00000000000..48b25f21f86 --- /dev/null +++ b/litellm/llms/bedrock/messages/native_transformation.py @@ -0,0 +1,131 @@ +import json +from collections.abc import Callable +from typing import Final + +import httpx + +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + AnthropicMessagesConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token +from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url +from litellm.types.router import GenericLiteLLMParams + +MessagesURLBuilder = Callable[[str | None, str | None, str], str] # mutable-ok: provider interface # rebind-ok: test capture +BearerTokenResolver = Callable[[str | None], str | None] # mutable-ok: provider interface # rebind-ok: test capture + + +class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig): + preserves_request_body: bool = True + + def __init__( + self, + custom_llm_provider: str = "bedrock", + url_builder: MessagesURLBuilder | None = None, + bearer_token_resolver: BearerTokenResolver = bedrock_bearer_token, + **kwargs: object, # kwargs-ok: provider interface + ) -> None: + BaseAWSLLM.__init__(self, **kwargs) + self._custom_llm_provider = custom_llm_provider + self._url_builder = url_builder + self._bearer_token_resolver = bearer_token_resolver + + @property + def custom_llm_provider(self) -> str: + return self._custom_llm_provider + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + stream: bool | None = None, + ) -> str: + region: Final = self._get_aws_region_name(optional_params=optional_params, model=model) + endpoint: Final = optional_params.get("aws_bedrock_runtime_endpoint") + if self._url_builder is not None: + return self._url_builder(api_base, endpoint, region) + runtime_endpoint: Final = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=endpoint, + aws_region_name=region, + )[0] + return f"{runtime_endpoint.rstrip('/')}/anthropic/v1/messages" + + def validate_anthropic_messages_environment( + self, + headers: dict, # mutable-ok: provider interface + model: str, + messages: list[dict], # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + litellm_params: dict, # mutable-ok: provider interface + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict, str | None]: # mutable-ok: provider interface + return ( + { # mutable-ok: provider interface + **headers, + "anthropic-version": headers.get("anthropic-version", DEFAULT_ANTHROPIC_API_VERSION), + "content-type": headers.get("content-type", "application/json"), + }, + api_base, + ) + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict], # mutable-ok: provider interface + anthropic_messages_optional_request_params: dict, # mutable-ok: provider interface + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: provider interface + ) -> dict: # mutable-ok: provider interface + return { # mutable-ok: provider interface + **anthropic_messages_optional_request_params, + "model": model, + "messages": messages, + } + + def sign_request( + self, + headers: dict, # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + request_data: dict, # mutable-ok: provider interface + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface + bearer: Final = self._bearer_token_resolver(api_key) + if bearer: + return {**headers, "Authorization": f"Bearer {bearer}"}, json.dumps(request_data).encode() # mutable-ok: provider interface + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def get_error_class( + self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers # mutable-ok: provider interface + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message, headers=headers) + + +def mantle_native_messages_config() -> AmazonBedrockNativeMessagesConfig: + from litellm.llms.bedrock_mantle.common_utils import resolve_mantle_bearer_token + + return AmazonBedrockNativeMessagesConfig( + custom_llm_provider="bedrock_mantle", + url_builder=build_mantle_messages_url, + bearer_token_resolver=resolve_mantle_bearer_token, + ) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py new file mode 100644 index 00000000000..5d1cd04ff7f --- /dev/null +++ b/litellm/llms/bedrock/responses/transformation.py @@ -0,0 +1,80 @@ +from typing import Final + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ResponseInputParam +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: provider interface + OpenAIResponsesAPIConfig.__init__(self) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK + + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict, # mutable-ok: provider interface + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: provider interface + ) -> dict: # mutable-ok: provider interface + request_params: Final = { # mutable-ok: provider interface + key: value + for key, value in response_api_optional_request_params.items() + if key not in self.aws_authentication_params + } + return { # mutable-ok: provider interface + **request_params, + "model": model, + "input": input, + } + + def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: # mutable-ok: provider interface + region: Final = self._get_aws_region_name(optional_params=litellm_params, model="") + endpoint_url: Final = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=litellm_params.get("aws_bedrock_runtime_endpoint"), + aws_region_name=region, + )[0] + return f"{endpoint_url.rstrip('/')}/openai/v1/responses" + + def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict: # mutable-ok: provider interface + return {**headers, "Content-Type": headers.get("Content-Type", "application/json")} # mutable-ok: provider interface + + def sign_request( + self, + headers: dict, # mutable-ok: provider interface + optional_params: dict, # mutable-ok: provider interface + request_data: dict, # mutable-ok: provider interface + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def get_error_class( + self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers # mutable-ok: provider interface + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/main.py b/litellm/main.py index 17edafcdfca..676cf7959f7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4055,6 +4055,10 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + from litellm.llms.bedrock.chat.openai_chat_completions_transformation import ( + AmazonBedrockOpenAIChatCompletionsConfig, + ) + if "aws_bedrock_client" in optional_params: verbose_logger.warning( "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." @@ -4073,7 +4077,27 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) - if bedrock_route == "claude_platform": + if isinstance(provider_config, AmazonBedrockOpenAIChatCompletionsConfig): + return base_llm_http_handler.completion( + model=model.removeprefix("bedrock/"), + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + elif bedrock_route == "claude_platform": provider_config = ProviderConfigManager.get_provider_chat_config( model=model, provider=LlmProviders.BEDROCK, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index a68cd02e61b..6d5b46d913b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -52,6 +52,7 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.secret_managers.main import get_secret_str from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders from litellm.utils import ( ProviderConfigManager, client, @@ -466,7 +467,9 @@ def _resolve_responses_api_provider_config( provider_config: Final = ProviderConfigManager.get_provider_responses_api_config( model=model, provider=custom_llm_provider ) - if provider_config is not None or not _deployment_passes_through_responses(model_info): + if provider_config is not None or not _deployment_supports_native_responses( + model_info, model=model, custom_llm_provider=custom_llm_provider + ): return provider_config return OpenAILikeResponsesConfig() @@ -2291,3 +2294,15 @@ async def _aresponses_websocket( custom_llm_provider=_custom_llm_provider, **remaining_kwargs, ) + + +def _deployment_supports_native_responses( + model_info: object, model: str | None, custom_llm_provider: str | None +) -> bool: + if _deployment_passes_through_responses(model_info): + return True + if model is None or custom_llm_provider not in tuple(provider.value for provider in LlmProviders): + return False + from litellm.utils import model_supports_native_endpoint + + return model_supports_native_endpoint("/v1/responses", model, LlmProviders(custom_llm_provider)) diff --git a/litellm/utils.py b/litellm/utils.py index c6386471f28..013d0393349 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8080,6 +8080,23 @@ def _get_model_cost_entry_for_provider_config( return {} +def model_supports_native_endpoint(endpoint: str, model: str, provider: LlmProviders) -> bool: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + base_model: Final = BedrockModelInfo.get_base_model(model) if provider is LlmProviders.BEDROCK else model + candidate_models: Final = (model, base_model) if base_model != model else (model,) + return any( + isinstance( + supported_endpoints := _get_model_cost_entry_for_provider_config(candidate_model, provider).get( + "supported_endpoints" + ), + (list, tuple), + ) + and endpoint in supported_endpoints + for candidate_model in candidate_models + ) + + class ProviderConfigManager: # Dictionary mapping for O(1) provider lookup # Stores tuples of (factory_function, needs_model_parameter) @@ -8538,6 +8555,20 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if provider is LlmProviders.BEDROCK and model_supports_native_endpoint( + "/v1/messages", model, LlmProviders.BEDROCK + ): + from litellm.llms.bedrock.messages.native_transformation import ( + AmazonBedrockNativeMessagesConfig, + ) + + return AmazonBedrockNativeMessagesConfig() + if provider is LlmProviders.BEDROCK_MANTLE and model_supports_native_endpoint( + "/v1/messages", model, LlmProviders.BEDROCK_MANTLE + ): + from litellm.llms.bedrock.messages.native_transformation import mantle_native_messages_config + + return mantle_native_messages_config() return ProviderConfigManager._get_provider_anthropic_messages_config_cached(model=model, provider=provider) @staticmethod @@ -8810,6 +8841,12 @@ class ProviderConfigManager: return litellm.BedrockMantleResponsesAPIConfig( use_openai_path=mantle_base_segment(model, litellm.model_cost) == "openai/v1" ) + elif litellm.LlmProviders.BEDROCK == provider: + if model is None or not model_supports_native_endpoint("/v1/responses", model, LlmProviders.BEDROCK): + return None + from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig + + return AmazonBedrockResponsesAPIConfig() return None @staticmethod diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py new file mode 100644 index 00000000000..c56a7dbdf8b --- /dev/null +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -0,0 +1,333 @@ +import json +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _deployment_supports_native_anthropic_messages, +) +from litellm.llms.bedrock.chat.openai_chat_completions_transformation import ( + AmazonBedrockOpenAIChatCompletionsConfig, +) +from litellm.llms.bedrock.common_utils import get_bedrock_chat_config +from litellm.llms.bedrock.messages.native_transformation import ( + AmazonBedrockNativeMessagesConfig, + mantle_native_messages_config, +) +from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig +from litellm.responses.main import _deployment_supports_native_responses +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, model_supports_native_endpoint + + +def _chat_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ # mutable-ok: provider interface + "id": "chat-test", + "object": "chat.completion", + "created": 1, + "model": "native-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], # mutable-ok: provider interface + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, # mutable-ok: provider interface + }, + request=httpx.Request("POST", url), + ) + + +def _messages_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ # mutable-ok: provider interface + "id": "message-test", + "type": "message", + "role": "assistant", + "model": "native-model", + "content": [{"type": "text", "text": "ok"}], # mutable-ok: provider interface + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, # mutable-ok: provider interface + }, + request=httpx.Request("POST", url), + ) + + +def _responses_response(url: str) -> httpx.Response: + return httpx.Response( + 200, + json={ # mutable-ok: provider interface + "id": "response-test", + "object": "response", + "created_at": 1, + "model": "native-model", + "output": [], # mutable-ok: provider interface + "status": "completed", + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, # mutable-ok: provider interface + }, + request=httpx.Request("POST", url), + ) + + +def test_native_endpoint_gate_reads_provider_model_cost(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert model_supports_native_endpoint( + "/v1/chat/completions", "native-model", LlmProviders.BEDROCK + ) + assert not model_supports_native_endpoint("/v1/messages", "native-model", LlmProviders.BEDROCK) + + +def test_native_endpoint_gate_uses_bedrock_base_model(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-3-7-sonnet", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + + assert model_supports_native_endpoint( + "/v1/messages", "us.anthropic.claude-3-7-sonnet", LlmProviders.BEDROCK + ) + + +def test_bedrock_chat_route_prefers_explicit_converse(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert isinstance(get_bedrock_chat_config("native-model"), AmazonBedrockOpenAIChatCompletionsConfig) + assert type(get_bedrock_chat_config("converse/native-model")).__name__ == "AmazonConverseConfig" + + +def test_native_chat_url_and_bearer_auth(monkeypatch): + config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture + + assert config.get_complete_url( + None, None, "native-model", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface + ) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") + headers, body = config.sign_request( + {}, # mutable-ok: provider interface + {"aws_region_name": "us-west-2"}, # mutable-ok: provider interface + {"model": "native-model"}, # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", + ) + assert headers["Authorization"] == "Bearer token" + assert body is not None + + +def test_native_chat_sigv4_auth(monkeypatch): + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture + headers, _ = config.sign_request( + {}, # mutable-ok: provider interface + { # mutable-ok: provider interface + "aws_region_name": "us-west-2", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing", + }, + {"model": "native-model"}, # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", + ) + + assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") + + +def test_native_messages_preserves_body_and_urls(): + config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture + request = { # mutable-ok: provider interface # rebind-ok: test capture + "model": "claude-native", + "messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface + "stream": True, + "cache_control": {"ttl": "5m"}, # mutable-ok: provider interface + "unknown_field": "kept", + } + body = config.transform_anthropic_messages_request( # rebind-ok: test capture + model=request["model"], + messages=request["messages"], + anthropic_messages_optional_request_params={ # mutable-ok: provider interface + "stream": request["stream"], + "cache_control": request["cache_control"], + "unknown_field": request["unknown_field"], + }, + litellm_params={}, # mutable-ok: provider interface + headers={}, # mutable-ok: provider interface + ) + + assert body == request + assert "anthropic_version" not in body + assert config.get_complete_url(None, None, "claude-native", {"aws_region_name": "us-west-2"}, {}) == ( # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages" + ) + assert mantle_native_messages_config().get_complete_url( + None, None, "claude-native", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface + ) == "https://bedrock-mantle.us-west-2.api.aws/anthropic/v1/messages" + + +def test_native_messages_selection_is_not_cached(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + assert isinstance( + ProviderConfigManager.get_provider_anthropic_messages_config( + "native-model", LlmProviders.BEDROCK + ), + AmazonBedrockNativeMessagesConfig, + ) + + monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface + assert type( + ProviderConfigManager.get_provider_anthropic_messages_config( + "claude-native", LlmProviders.BEDROCK + ) + ).__name__ == "AmazonAnthropicClaudeMessagesConfig" + + +def test_native_responses_selection_and_url(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/responses"]}, # mutable-ok: provider interface + ) + config = ProviderConfigManager.get_provider_responses_api_config( # rebind-ok: test capture + LlmProviders.BEDROCK, "native-model" + ) + + assert isinstance(config, AmazonBedrockResponsesAPIConfig) + assert config.get_complete_url(None, {"aws_region_name": "us-west-2"}).endswith( # mutable-ok: provider interface + "/openai/v1/responses" + ) + + monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface + assert ProviderConfigManager.get_provider_responses_api_config( + LlmProviders.BEDROCK, "native-model" + ) is None + + +def test_generic_endpoint_helpers_read_cost_map(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "hosted_vllm/native-model", + {"supported_endpoints": ["/v1/messages", "/v1/responses"]}, # mutable-ok: provider interface + ) + + assert _deployment_supports_native_anthropic_messages( + None, model="native-model", custom_llm_provider="hosted_vllm" + ) + assert _deployment_supports_native_responses( + None, model="native-model", custom_llm_provider="hosted_vllm" + ) + + +def test_completion_forwards_native_chat_body(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _chat_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.completion( + model="bedrock/native-model", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests == [ # mutable-ok: provider interface + ( + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", + {"model": "native-model", "messages": [{"role": "user", "content": "hello"}]}, # mutable-ok: provider interface + ) + ] + + +@pytest.mark.asyncio +async def test_anthropic_messages_forwards_native_body(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + async def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _messages_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=post + ): + await litellm.anthropic_messages( + model="bedrock/native-model", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + max_tokens=10, + unknown_field="kept", + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests == [ # mutable-ok: provider interface + ( + "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages", + { # mutable-ok: provider interface + "max_tokens": 10, + "stream": False, + "unknown_field": "kept", + "model": "native-model", + "messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface + }, + ) + ] + + +def test_responses_forwards_native_body(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/native-model", + {"supported_endpoints": ["/v1/responses"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _responses_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.responses( + model="bedrock/native-model", + input="hello", + instructions="be brief", + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests == [ # mutable-ok: provider interface + ( + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses", + { # mutable-ok: provider interface + "model": "native-model", + "input": "hello", + "instructions": "be brief", + }, + ) + ] From 21a72defa573bde60f1415348c2a1d9e3fc1cc25 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 03:06:52 +0000 Subject: [PATCH 2/9] fix(bedrock): simplify native messages forwarding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/handler.py | 8 +-- .../messages/native_utils.py | 51 ------------------- .../openai_chat_completions_transformation.py | 5 +- .../bedrock/messages/native_transformation.py | 25 ++++----- .../llms/bedrock/responses/transformation.py | 14 +++-- .../test_native_endpoint_forwarding.py | 15 ++++-- 6 files changed, 39 insertions(+), 79 deletions(-) delete mode 100644 litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 2bf61bb65fb..e17b431cf34 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -38,7 +38,6 @@ from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from ..utils import is_reasoning_auto_summary_enabled from .interceptors import get_messages_interceptors -from .native_utils import get_native_messages_request_params from .utils import AnthropicMessagesRequestUtils, mock_response # Providers that are routed directly to the OpenAI Responses API instead of @@ -642,11 +641,6 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, ) ) - forwarded_anthropic_messages_optional_request_params: Final = get_native_messages_request_params( - optional_params=anthropic_messages_optional_request_params, - kwargs=kwargs, - preserves_request_body=getattr(anthropic_messages_provider_config, "preserves_request_body", False), - ) if is_reasoning_auto_summary_enabled(): thinking_param: Final = anthropic_messages_optional_request_params.get("thinking") if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled": @@ -659,7 +653,7 @@ def anthropic_messages_handler( model=model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=forwarded_anthropic_messages_optional_request_params, + anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params), _is_async=is_async, client=client, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py deleted file mode 100644 index 7d4e1b6df6f..00000000000 --- a/litellm/llms/anthropic/experimental_pass_through/messages/native_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final - -_NATIVE_REQUEST_EXCLUDED_KEYS: Final = frozenset( - { - "api_base", - "api_key", - "client", - "custom_llm_provider", - "is_async", - "kwargs", - "litellm_logging_obj", - "litellm_call_id", - "model", - "messages", - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - "aws_region_name", - "aws_session_name", - "aws_profile_name", - "aws_role_name", - "aws_web_identity_token", - "aws_sts_endpoint", - "aws_bedrock_runtime_endpoint", - "aws_external_id", - "aws_session_tags", - } -) - - -def get_native_messages_request_params( - optional_params: Mapping[str, object], - kwargs: Mapping[str, object], - preserves_request_body: bool, -) -> Mapping[str, object]: - if not preserves_request_body: - return optional_params - return MappingProxyType( - { - **optional_params, - **MappingProxyType( - { - key: value - for key, value in kwargs.items() - if key not in _NATIVE_REQUEST_EXCLUDED_KEYS and value is not None - } - ), - } - ) diff --git a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py index 1ddf7b5dd12..7d23f837334 100644 --- a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py +++ b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py @@ -99,6 +99,9 @@ class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM): ) def get_error_class( - self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers # mutable-ok: provider interface + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface ) -> BedrockError: return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/bedrock/messages/native_transformation.py b/litellm/llms/bedrock/messages/native_transformation.py index 48b25f21f86..5571013d193 100644 --- a/litellm/llms/bedrock/messages/native_transformation.py +++ b/litellm/llms/bedrock/messages/native_transformation.py @@ -1,6 +1,4 @@ -import json -from collections.abc import Callable -from typing import Final +from typing import Final, Protocol import httpx @@ -13,13 +11,16 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url from litellm.types.router import GenericLiteLLMParams -MessagesURLBuilder = Callable[[str | None, str | None, str], str] # mutable-ok: provider interface # rebind-ok: test capture -BearerTokenResolver = Callable[[str | None], str | None] # mutable-ok: provider interface # rebind-ok: test capture + +class MessagesURLBuilder(Protocol): + def __call__(self, api_base: str | None, endpoint: str | None, region: str) -> str: ... + + +class BearerTokenResolver(Protocol): + def __call__(self, api_key: str | None) -> str | None: ... class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig): - preserves_request_body: bool = True - def __init__( self, custom_llm_provider: str = "bedrock", @@ -100,23 +101,23 @@ class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig): stream: bool | None = None, fake_stream: bool | None = None, ) -> tuple[dict, bytes | None]: # mutable-ok: provider interface - bearer: Final = self._bearer_token_resolver(api_key) - if bearer: - return {**headers, "Authorization": f"Bearer {bearer}"}, json.dumps(request_data).encode() # mutable-ok: provider interface return self._sign_request( service_name="bedrock", headers=headers, optional_params=optional_params, request_data=request_data, api_base=api_base, - api_key=api_key, + api_key=self._bearer_token_resolver(api_key), model=model, stream=stream, fake_stream=fake_stream, ) def get_error_class( - self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers # mutable-ok: provider interface + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface ) -> BaseLLMException: return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 5d1cd04ff7f..e2a26245a29 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -48,8 +48,13 @@ class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM): )[0] return f"{endpoint_url.rstrip('/')}/openai/v1/responses" - def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict: # mutable-ok: provider interface - return {**headers, "Content-Type": headers.get("Content-Type", "application/json")} # mutable-ok: provider interface + def validate_environment( + self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict: # mutable-ok: provider interface + return { + **headers, + "Content-Type": headers.get("Content-Type", "application/json"), + } # mutable-ok: provider interface def sign_request( self, @@ -75,6 +80,9 @@ class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM): ) def get_error_class( - self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers # mutable-ok: provider interface + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: provider interface ) -> BaseLLMException: return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py index c56a7dbdf8b..571b62ed361 100644 --- a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -141,14 +141,14 @@ def test_native_chat_sigv4_auth(monkeypatch): assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") -def test_native_messages_preserves_body_and_urls(): +def test_native_messages_preserves_body_and_urls(monkeypatch): + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture request = { # mutable-ok: provider interface # rebind-ok: test capture "model": "claude-native", "messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface "stream": True, "cache_control": {"ttl": "5m"}, # mutable-ok: provider interface - "unknown_field": "kept", } body = config.transform_anthropic_messages_request( # rebind-ok: test capture model=request["model"], @@ -156,7 +156,6 @@ def test_native_messages_preserves_body_and_urls(): anthropic_messages_optional_request_params={ # mutable-ok: provider interface "stream": request["stream"], "cache_control": request["cache_control"], - "unknown_field": request["unknown_field"], }, litellm_params={}, # mutable-ok: provider interface headers={}, # mutable-ok: provider interface @@ -170,6 +169,14 @@ def test_native_messages_preserves_body_and_urls(): assert mantle_native_messages_config().get_complete_url( None, None, "claude-native", {"aws_region_name": "us-west-2"}, {} # mutable-ok: provider interface ) == "https://bedrock-mantle.us-west-2.api.aws/anthropic/v1/messages" + headers, signed_body = config.sign_request( + {}, # mutable-ok: provider interface + {"aws_region_name": "us-west-2"}, # mutable-ok: provider interface + body, # mutable-ok: provider interface + "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages", + ) + assert headers["Authorization"] == "Bearer token" + assert signed_body is not None def test_native_messages_selection_is_not_cached(monkeypatch): @@ -279,7 +286,6 @@ async def test_anthropic_messages_forwards_native_body(monkeypatch): model="bedrock/native-model", messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface max_tokens=10, - unknown_field="kept", api_base="https://bedrock-runtime.us-west-2.amazonaws.com", api_key="token", ) @@ -290,7 +296,6 @@ async def test_anthropic_messages_forwards_native_body(monkeypatch): { # mutable-ok: provider interface "max_tokens": 10, "stream": False, - "unknown_field": "kept", "model": "native-model", "messages": [{"role": "user", "content": "hello"}], # mutable-ok: provider interface }, From 28079dd187d5c3c2bbff96602113d17f65a9f910 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 03:35:03 +0000 Subject: [PATCH 3/9] fix: honor native chat gate for converse models and dedupe content-type Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../openai_chat_completions_transformation.py | 8 +- litellm/llms/bedrock/common_utils.py | 80 ++++++++++++------- .../bedrock/messages/native_transformation.py | 8 +- .../llms/bedrock/responses/transformation.py | 7 +- litellm/utils.py | 14 +++- .../test_native_endpoint_forwarding.py | 71 +++++++++++++++- 6 files changed, 149 insertions(+), 39 deletions(-) diff --git a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py index 7d23f837334..279bafe1081 100644 --- a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py +++ b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py @@ -69,7 +69,13 @@ class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM): ) -> dict: # mutable-ok: provider interface owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params) return merge_bedrock_invoke_headers( - headers, + { + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "Content-Type": next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ), + }, (), metadata_headers, owned_names, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9f5656dd455..d9d62d9c5e9 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -35,6 +35,43 @@ if TYPE_CHECKING: _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" +_BEDROCK_ROUTE_PREFIXES: Final[tuple[str, ...]] = ( + "invoke/", + "claude_platform/", + "converse_like/", + "converse/", + "agent/", + "agentcore/", + "async_invoke/", + "openai/", + "mantle/", +) +_BEDROCK_ROUTE_TYPES: Final[ + tuple[ + Literal[ + "invoke", + "claude_platform", + "converse_like", + "converse", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", + ], + ..., + ] +] = ( + "invoke", + "claude_platform", + "converse_like", + "converse", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", +) def error_response_text(response: httpx.Response) -> str: @@ -1090,6 +1127,10 @@ class BedrockModelInfo(BaseLLMModelInfo): """Wrapper for standalone function. See get_bedrock_base_model().""" return get_bedrock_base_model(model) + @staticmethod + def has_explicit_route(model: str) -> bool: + return any(BedrockModelInfo._model_has_route_prefix(model, prefix) for prefix in _BEDROCK_ROUTE_PREFIXES) + @staticmethod def _supported_cross_region_inference_region() -> list[str]: """Wrapper for standalone function. See get_bedrock_cross_region_inference_regions().""" @@ -1112,36 +1153,11 @@ class BedrockModelInfo(BaseLLMModelInfo): """ Get the bedrock route for the given model. """ - route_mappings: dict[ - str, - Literal[ - "invoke", - "claude_platform", - "converse_like", - "converse", - "agent", - "agentcore", - "async_invoke", - "openai", - "mantle", - ], - ] = { - "invoke/": "invoke", - "claude_platform/": "claude_platform", - "converse_like/": "converse_like", - "converse/": "converse", - "agent/": "agent", - "agentcore/": "agentcore", - "async_invoke/": "async_invoke", - "openai/": "openai", - "mantle/": "mantle", - } - # Check explicit routes first. Match each prefix only as a leading path # segment so the `bedrock_mantle/` provider prefix is never mistaken for # the `mantle/` invoke route (which would mangle # `bedrock_mantle/openai.gpt-5.5` into `bedrock_openai.gpt-5.5`). - for prefix, route_type in route_mappings.items(): + for prefix, route_type in zip(_BEDROCK_ROUTE_PREFIXES, _BEDROCK_ROUTE_TYPES, strict=True): if BedrockModelInfo._model_has_route_prefix(model, prefix): return route_type @@ -1331,6 +1347,13 @@ def get_bedrock_chat_config(model: str): bedrock_invoke_provider: Final = BaseAWSLLM.get_bedrock_invoke_provider(model=model) base_model: Final = BedrockModelInfo.get_base_model(model) + from litellm.utils import model_supports_native_endpoint + + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/chat/completions", model, LlmProviders.BEDROCK + ): + return litellm.AmazonBedrockOpenAIChatCompletionsConfig() + # Handle explicit routes first if bedrock_route == "claude_platform": return litellm.BedrockClaudePlatformConfig() @@ -1357,11 +1380,6 @@ def get_bedrock_chat_config(model: str): return AmazonMantleConfig() - from litellm.utils import model_supports_native_endpoint - - if model_supports_native_endpoint("/v1/chat/completions", model, LlmProviders.BEDROCK): - return litellm.AmazonBedrockOpenAIChatCompletionsConfig() - # Handle provider-specific configs if bedrock_invoke_provider == "amazon": return litellm.AmazonTitanConfig() diff --git a/litellm/llms/bedrock/messages/native_transformation.py b/litellm/llms/bedrock/messages/native_transformation.py index 5571013d193..ca481851ced 100644 --- a/litellm/llms/bedrock/messages/native_transformation.py +++ b/litellm/llms/bedrock/messages/native_transformation.py @@ -67,11 +67,15 @@ class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig): api_key: str | None = None, api_base: str | None = None, ) -> tuple[dict, str | None]: # mutable-ok: provider interface + content_type: Final = next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ) return ( { # mutable-ok: provider interface - **headers, + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, "anthropic-version": headers.get("anthropic-version", DEFAULT_ANTHROPIC_API_VERSION), - "content-type": headers.get("content-type", "application/json"), + "Content-Type": content_type, }, api_base, ) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index e2a26245a29..8fc065fcde7 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,8 +52,11 @@ class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM): self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None ) -> dict: # mutable-ok: provider interface return { - **headers, - "Content-Type": headers.get("Content-Type", "application/json"), + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "Content-Type": next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ), } # mutable-ok: provider interface def sign_request( diff --git a/litellm/utils.py b/litellm/utils.py index 013d0393349..ac62b6fbadf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4489,7 +4489,17 @@ def get_optional_params( BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo") bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) bedrock_base_model: Final = BedrockModelInfo.get_base_model(model) - if bedrock_route == "converse" or bedrock_route == "converse_like": + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/chat/completions", model, LlmProviders.BEDROCK + ): + native_provider_config: Final = provider_config or litellm.AmazonBedrockOpenAIChatCompletionsConfig() + optional_params = native_provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + ) + elif bedrock_route == "converse" or bedrock_route == "converse_like": optional_params = litellm.AmazonConverseConfig().map_openai_params( model=model, non_default_params=non_default_params, @@ -8066,7 +8076,7 @@ def _get_model_cost_entry_for_provider_config( model: str, provider: LlmProviders, ) -> dict[str, Any]: - candidate_keys: Final = (model, f"{provider.value}/{model}") + candidate_keys: Final = (f"{provider.value}/{model}", model) for model_key in candidate_keys: model_info = litellm.model_cost.get(model_key) if model_info is not None: diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py index 571b62ed361..117179cebe4 100644 --- a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -106,6 +106,20 @@ def test_bedrock_chat_route_prefers_explicit_converse(monkeypatch): assert type(get_bedrock_chat_config("converse/native-model")).__name__ == "AmazonConverseConfig" +def test_bedrock_converse_catalog_model_prefers_declared_native_chat(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert isinstance( + get_bedrock_chat_config("anthropic.claude-sonnet-4-6"), + AmazonBedrockOpenAIChatCompletionsConfig, + ) + assert type(get_bedrock_chat_config("converse/anthropic.claude-sonnet-4-6")).__name__ == "AmazonConverseConfig" + + def test_native_chat_url_and_bearer_auth(monkeypatch): config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture @@ -114,13 +128,21 @@ def test_native_chat_url_and_bearer_auth(monkeypatch): ) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") - headers, body = config.sign_request( + validated_headers = config.validate_environment( + {"content-type": "application/json"}, # mutable-ok: provider interface + "native-model", + [], # mutable-ok: provider interface {}, # mutable-ok: provider interface + {}, # mutable-ok: provider interface + ) + headers, body = config.sign_request( + validated_headers, {"aws_region_name": "us-west-2"}, # mutable-ok: provider interface {"model": "native-model"}, # mutable-ok: provider interface "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", ) assert headers["Authorization"] == "Bearer token" + assert sum(key.lower() == "content-type" for key in headers) == 1 assert body is not None @@ -176,9 +198,25 @@ def test_native_messages_preserves_body_and_urls(monkeypatch): "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages", ) assert headers["Authorization"] == "Bearer token" + assert sum(key.lower() == "content-type" for key in headers) == 1 assert signed_body is not None +def test_native_messages_deduplicates_content_type_header(monkeypatch): + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") + config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture + headers, _ = config.validate_anthropic_messages_environment( + headers={"content-type": "application/json"}, # mutable-ok: provider interface + model="native-model", + messages=[], # mutable-ok: provider interface + optional_params={}, # mutable-ok: provider interface + litellm_params={}, # mutable-ok: provider interface + ) + + assert sum(key.lower() == "content-type" for key in headers) == 1 + assert headers["Content-Type"] == "application/json" + + def test_native_messages_selection_is_not_cached(monkeypatch): monkeypatch.setitem( litellm.model_cost, @@ -214,6 +252,12 @@ def test_native_responses_selection_and_url(monkeypatch): assert config.get_complete_url(None, {"aws_region_name": "us-west-2"}).endswith( # mutable-ok: provider interface "/openai/v1/responses" ) + validated_headers = config.validate_environment( + {"content-type": "application/json"}, # mutable-ok: provider interface + "native-model", + {}, # mutable-ok: provider interface + ) + assert sum(key.lower() == "content-type" for key in validated_headers) == 1 monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface assert ProviderConfigManager.get_provider_responses_api_config( @@ -266,6 +310,31 @@ def test_completion_forwards_native_chat_body(monkeypatch): ] +def test_completion_forwards_declared_native_chat_for_converse_catalog_model(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _chat_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.completion( + model="bedrock/anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests[0][0].endswith("/openai/v1/chat/completions") + + @pytest.mark.asyncio async def test_anthropic_messages_forwards_native_body(monkeypatch): monkeypatch.setitem( From debf90e56823af391058cab3f3a1a777f8b47f6a Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 03:57:54 +0000 Subject: [PATCH 4/9] test(responses): avoid native metadata model in prompt tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/test_responses_prompt_management.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 530afbd856b..4dba303073a 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -320,7 +320,7 @@ class TestResponsesAPIPromptManagement: def test_cache_control_hook_preserves_reasoning_items(self): original_input, merged_messages, reasoning_item = _make_cache_control_case() logging_obj = _make_logging_obj( - merged_model="azure/gpt-5-codex", + merged_model="azure/prompt-management-test-model", merged_messages=merged_messages, ) @@ -330,7 +330,7 @@ class TestResponsesAPIPromptManagement: litellm.responses( input=original_input, - model="azure/gpt-5-codex", + model="azure/prompt-management-test-model", litellm_logging_obj=logging_obj, cache_control_injection_points=[{"location": "message", "role": "system"}], ) @@ -516,7 +516,7 @@ class TestAsyncResponsesAPIPromptManagement: async def test_async_cache_control_hook_preserves_reasoning_items(self): original_input, merged_messages, reasoning_item = _make_cache_control_case() logging_obj = _make_logging_obj( - merged_model="azure/gpt-5-codex", + merged_model="azure/prompt-management-test-model", merged_messages=merged_messages, ) @@ -526,7 +526,7 @@ class TestAsyncResponsesAPIPromptManagement: await litellm.aresponses( input=original_input, - model="azure/gpt-5-codex", + model="azure/prompt-management-test-model", litellm_logging_obj=logging_obj, cache_control_injection_points=[{"location": "message", "role": "system"}], ) From 5d41295c2a96404a49ece425f75f56a8ac1fd9e1 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 04:15:07 +0000 Subject: [PATCH 5/9] fix: preserve explicit native endpoint opt-in Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/handler.py | 16 ++++------------ litellm/responses/main.py | 17 +++-------------- .../test_native_endpoint_forwarding.py | 19 ------------------- .../test_responses_prompt_management.py | 8 ++++---- 4 files changed, 11 insertions(+), 49 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index e17b431cf34..0548abaa2ff 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -31,7 +31,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import CallTypes, LlmProviders +from litellm.types.utils import CallTypes from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler @@ -567,7 +567,7 @@ def anthropic_messages_handler( provider=litellm.LlmProviders(custom_llm_provider), ) if anthropic_messages_provider_config is None and _deployment_supports_native_anthropic_messages( - kwargs.get("model_info"), model=model, custom_llm_provider=custom_llm_provider + kwargs.get("model_info") ): from litellm.llms.openai_like.messages.transformation import ( OpenAILikeAnthropicMessagesConfig, @@ -666,13 +666,5 @@ def anthropic_messages_handler( ) -def _deployment_supports_native_anthropic_messages( - model_info: object, model: str | None, custom_llm_provider: str | None -) -> bool: - if _deployment_passes_through_anthropic_messages(model_info): - return True - if model is None or custom_llm_provider not in tuple(provider.value for provider in LlmProviders): - return False - from litellm.utils import model_supports_native_endpoint - - return model_supports_native_endpoint("/v1/messages", model, LlmProviders(custom_llm_provider)) +def _deployment_supports_native_anthropic_messages(model_info: object) -> bool: + return _deployment_passes_through_anthropic_messages(model_info) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 6d5b46d913b..73bbb334537 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -52,7 +52,6 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.secret_managers.main import get_secret_str from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import LlmProviders from litellm.utils import ( ProviderConfigManager, client, @@ -467,9 +466,7 @@ def _resolve_responses_api_provider_config( provider_config: Final = ProviderConfigManager.get_provider_responses_api_config( model=model, provider=custom_llm_provider ) - if provider_config is not None or not _deployment_supports_native_responses( - model_info, model=model, custom_llm_provider=custom_llm_provider - ): + if provider_config is not None or not _deployment_supports_native_responses(model_info): return provider_config return OpenAILikeResponsesConfig() @@ -2296,13 +2293,5 @@ async def _aresponses_websocket( ) -def _deployment_supports_native_responses( - model_info: object, model: str | None, custom_llm_provider: str | None -) -> bool: - if _deployment_passes_through_responses(model_info): - return True - if model is None or custom_llm_provider not in tuple(provider.value for provider in LlmProviders): - return False - from litellm.utils import model_supports_native_endpoint - - return model_supports_native_endpoint("/v1/responses", model, LlmProviders(custom_llm_provider)) +def _deployment_supports_native_responses(model_info: object) -> bool: + return _deployment_passes_through_responses(model_info) diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py index 117179cebe4..a577673a15a 100644 --- a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -5,9 +5,6 @@ import httpx import pytest import litellm -from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( - _deployment_supports_native_anthropic_messages, -) from litellm.llms.bedrock.chat.openai_chat_completions_transformation import ( AmazonBedrockOpenAIChatCompletionsConfig, ) @@ -17,7 +14,6 @@ from litellm.llms.bedrock.messages.native_transformation import ( mantle_native_messages_config, ) from litellm.llms.bedrock.responses.transformation import AmazonBedrockResponsesAPIConfig -from litellm.responses.main import _deployment_supports_native_responses from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, model_supports_native_endpoint @@ -265,21 +261,6 @@ def test_native_responses_selection_and_url(monkeypatch): ) is None -def test_generic_endpoint_helpers_read_cost_map(monkeypatch): - monkeypatch.setitem( - litellm.model_cost, - "hosted_vllm/native-model", - {"supported_endpoints": ["/v1/messages", "/v1/responses"]}, # mutable-ok: provider interface - ) - - assert _deployment_supports_native_anthropic_messages( - None, model="native-model", custom_llm_provider="hosted_vllm" - ) - assert _deployment_supports_native_responses( - None, model="native-model", custom_llm_provider="hosted_vllm" - ) - - def test_completion_forwards_native_chat_body(monkeypatch): monkeypatch.setitem( litellm.model_cost, diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 4dba303073a..530afbd856b 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -320,7 +320,7 @@ class TestResponsesAPIPromptManagement: def test_cache_control_hook_preserves_reasoning_items(self): original_input, merged_messages, reasoning_item = _make_cache_control_case() logging_obj = _make_logging_obj( - merged_model="azure/prompt-management-test-model", + merged_model="azure/gpt-5-codex", merged_messages=merged_messages, ) @@ -330,7 +330,7 @@ class TestResponsesAPIPromptManagement: litellm.responses( input=original_input, - model="azure/prompt-management-test-model", + model="azure/gpt-5-codex", litellm_logging_obj=logging_obj, cache_control_injection_points=[{"location": "message", "role": "system"}], ) @@ -516,7 +516,7 @@ class TestAsyncResponsesAPIPromptManagement: async def test_async_cache_control_hook_preserves_reasoning_items(self): original_input, merged_messages, reasoning_item = _make_cache_control_case() logging_obj = _make_logging_obj( - merged_model="azure/prompt-management-test-model", + merged_model="azure/gpt-5-codex", merged_messages=merged_messages, ) @@ -526,7 +526,7 @@ class TestAsyncResponsesAPIPromptManagement: await litellm.aresponses( input=original_input, - model="azure/prompt-management-test-model", + model="azure/gpt-5-codex", litellm_logging_obj=logging_obj, cache_control_injection_points=[{"location": "message", "role": "system"}], ) From e100fe924a67dfc26843a52bce3e048955356a07 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 04:15:49 +0000 Subject: [PATCH 6/9] refactor: drop unused native endpoint aliases in responses and messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../anthropic/experimental_pass_through/messages/handler.py | 6 +----- litellm/responses/main.py | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 0548abaa2ff..9d1e921cce4 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -566,7 +566,7 @@ def anthropic_messages_handler( model=model, provider=litellm.LlmProviders(custom_llm_provider), ) - if anthropic_messages_provider_config is None and _deployment_supports_native_anthropic_messages( + if anthropic_messages_provider_config is None and _deployment_passes_through_anthropic_messages( kwargs.get("model_info") ): from litellm.llms.openai_like.messages.transformation import ( @@ -664,7 +664,3 @@ def anthropic_messages_handler( stream=stream, kwargs=kwargs, ) - - -def _deployment_supports_native_anthropic_messages(model_info: object) -> bool: - return _deployment_passes_through_anthropic_messages(model_info) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 73bbb334537..a68cd02e61b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -466,7 +466,7 @@ def _resolve_responses_api_provider_config( provider_config: Final = ProviderConfigManager.get_provider_responses_api_config( model=model, provider=custom_llm_provider ) - if provider_config is not None or not _deployment_supports_native_responses(model_info): + if provider_config is not None or not _deployment_passes_through_responses(model_info): return provider_config return OpenAILikeResponsesConfig() @@ -2291,7 +2291,3 @@ async def _aresponses_websocket( custom_llm_provider=_custom_llm_provider, **remaining_kwargs, ) - - -def _deployment_supports_native_responses(model_info: object) -> bool: - return _deployment_passes_through_responses(model_info) From 3081b8ac7a8ec3b5dff5c950a70e26dd50fbfee1 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 05:06:10 +0000 Subject: [PATCH 7/9] fix(bedrock): avoid native endpoint import cycles Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/common_utils.py | 5 ++--- litellm/utils.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index d9d62d9c5e9..23d24aaffbf 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -28,7 +28,6 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret, get_secret_str -from litellm.types.utils import LlmProviders if TYPE_CHECKING: from litellm.types.llms.openai import AllMessageValues @@ -1311,7 +1310,7 @@ class BedrockModelInfo(BaseLLMModelInfo): from litellm.utils import model_supports_native_endpoint - if model_supports_native_endpoint("/v1/messages", model, LlmProviders.BEDROCK): + if model_supports_native_endpoint("/v1/messages", model, litellm.LlmProviders.BEDROCK): from litellm.llms.bedrock.messages.native_transformation import ( AmazonBedrockNativeMessagesConfig, ) @@ -1350,7 +1349,7 @@ def get_bedrock_chat_config(model: str): from litellm.utils import model_supports_native_endpoint if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( - "/v1/chat/completions", model, LlmProviders.BEDROCK + "/v1/chat/completions", model, litellm.LlmProviders.BEDROCK ): return litellm.AmazonBedrockOpenAIChatCompletionsConfig() diff --git a/litellm/utils.py b/litellm/utils.py index 38b65179a87..b064789b32b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8100,9 +8100,8 @@ def _get_model_cost_entry_for_provider_config( def model_supports_native_endpoint(endpoint: str, model: str, provider: LlmProviders) -> bool: - from litellm.llms.bedrock.common_utils import BedrockModelInfo - - base_model: Final = BedrockModelInfo.get_base_model(model) if provider is LlmProviders.BEDROCK else model + bedrock_model_info: Final = getattr(litellm, "BedrockModelInfo") + base_model: Final = bedrock_model_info.get_base_model(model) if provider is LlmProviders.BEDROCK else model candidate_models: Final = (model, base_model) if base_model != model else (model,) return any( isinstance( From 645dec3a3034d98fb2371e767fb0192fee77f5c8 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 05:08:24 +0000 Subject: [PATCH 8/9] fix(bedrock): satisfy strict import-cycle lint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index b064789b32b..30473853926 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8100,7 +8100,7 @@ def _get_model_cost_entry_for_provider_config( def model_supports_native_endpoint(endpoint: str, model: str, provider: LlmProviders) -> bool: - bedrock_model_info: Final = getattr(litellm, "BedrockModelInfo") + bedrock_model_info: Final = litellm.BedrockModelInfo base_model: Final = bedrock_model_info.get_base_model(model) if provider is LlmProviders.BEDROCK else model candidate_models: Final = (model, base_model) if base_model != model else (model,) return any( From 29c6bebae015140c9d4a01dee38a5da904fecf67 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 05:23:05 +0000 Subject: [PATCH 9/9] fix(bedrock): keep explicit routes authoritative for native messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/common_utils.py | 4 +++- litellm/utils.py | 6 +++-- .../test_native_endpoint_forwarding.py | 22 +++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 23d24aaffbf..d0a64662d8e 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1310,7 +1310,9 @@ class BedrockModelInfo(BaseLLMModelInfo): from litellm.utils import model_supports_native_endpoint - if model_supports_native_endpoint("/v1/messages", model, litellm.LlmProviders.BEDROCK): + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/messages", model, litellm.LlmProviders.BEDROCK + ): from litellm.llms.bedrock.messages.native_transformation import ( AmazonBedrockNativeMessagesConfig, ) diff --git a/litellm/utils.py b/litellm/utils.py index 30473853926..8039d4ec846 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8573,8 +8573,10 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: - if provider is LlmProviders.BEDROCK and model_supports_native_endpoint( - "/v1/messages", model, LlmProviders.BEDROCK + if ( + provider is LlmProviders.BEDROCK + and not litellm.BedrockModelInfo.has_explicit_route(model) + and model_supports_native_endpoint("/v1/messages", model, LlmProviders.BEDROCK) ): from litellm.llms.bedrock.messages.native_transformation import ( AmazonBedrockNativeMessagesConfig, diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py index a577673a15a..138f861eeb7 100644 --- a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -116,6 +116,28 @@ def test_bedrock_converse_catalog_model_prefers_declared_native_chat(monkeypatch assert type(get_bedrock_chat_config("converse/anthropic.claude-sonnet-4-6")).__name__ == "AmazonConverseConfig" +def test_bedrock_messages_route_prefers_explicit_invoke(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/messages"]}, # mutable-ok: provider interface + ) + + assert isinstance( + ProviderConfigManager.get_provider_anthropic_messages_config( + model="anthropic.claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ), + AmazonBedrockNativeMessagesConfig, + ) + assert type( + ProviderConfigManager.get_provider_anthropic_messages_config( + model="invoke/anthropic.claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + ).__name__ == "AmazonAnthropicClaudeMessagesConfig" + + def test_native_chat_url_and_bearer_auth(monkeypatch): config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture