From 28079dd187d5c3c2bbff96602113d17f65a9f910 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 03:35:03 +0000 Subject: [PATCH] fix: honor native chat gate for converse models and dedupe content-type Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../openai_chat_completions_transformation.py | 8 +- litellm/llms/bedrock/common_utils.py | 80 ++++++++++++------- .../bedrock/messages/native_transformation.py | 8 +- .../llms/bedrock/responses/transformation.py | 7 +- litellm/utils.py | 14 +++- .../test_native_endpoint_forwarding.py | 71 +++++++++++++++- 6 files changed, 149 insertions(+), 39 deletions(-) diff --git a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py index 7d23f837334..279bafe1081 100644 --- a/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py +++ b/litellm/llms/bedrock/chat/openai_chat_completions_transformation.py @@ -69,7 +69,13 @@ class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM): ) -> dict: # mutable-ok: provider interface owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params) return merge_bedrock_invoke_headers( - headers, + { + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "Content-Type": next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ), + }, (), metadata_headers, owned_names, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9f5656dd455..d9d62d9c5e9 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -35,6 +35,43 @@ if TYPE_CHECKING: _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" +_BEDROCK_ROUTE_PREFIXES: Final[tuple[str, ...]] = ( + "invoke/", + "claude_platform/", + "converse_like/", + "converse/", + "agent/", + "agentcore/", + "async_invoke/", + "openai/", + "mantle/", +) +_BEDROCK_ROUTE_TYPES: Final[ + tuple[ + Literal[ + "invoke", + "claude_platform", + "converse_like", + "converse", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", + ], + ..., + ] +] = ( + "invoke", + "claude_platform", + "converse_like", + "converse", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", +) def error_response_text(response: httpx.Response) -> str: @@ -1090,6 +1127,10 @@ class BedrockModelInfo(BaseLLMModelInfo): """Wrapper for standalone function. See get_bedrock_base_model().""" return get_bedrock_base_model(model) + @staticmethod + def has_explicit_route(model: str) -> bool: + return any(BedrockModelInfo._model_has_route_prefix(model, prefix) for prefix in _BEDROCK_ROUTE_PREFIXES) + @staticmethod def _supported_cross_region_inference_region() -> list[str]: """Wrapper for standalone function. See get_bedrock_cross_region_inference_regions().""" @@ -1112,36 +1153,11 @@ class BedrockModelInfo(BaseLLMModelInfo): """ Get the bedrock route for the given model. """ - route_mappings: dict[ - str, - Literal[ - "invoke", - "claude_platform", - "converse_like", - "converse", - "agent", - "agentcore", - "async_invoke", - "openai", - "mantle", - ], - ] = { - "invoke/": "invoke", - "claude_platform/": "claude_platform", - "converse_like/": "converse_like", - "converse/": "converse", - "agent/": "agent", - "agentcore/": "agentcore", - "async_invoke/": "async_invoke", - "openai/": "openai", - "mantle/": "mantle", - } - # Check explicit routes first. Match each prefix only as a leading path # segment so the `bedrock_mantle/` provider prefix is never mistaken for # the `mantle/` invoke route (which would mangle # `bedrock_mantle/openai.gpt-5.5` into `bedrock_openai.gpt-5.5`). - for prefix, route_type in route_mappings.items(): + for prefix, route_type in zip(_BEDROCK_ROUTE_PREFIXES, _BEDROCK_ROUTE_TYPES, strict=True): if BedrockModelInfo._model_has_route_prefix(model, prefix): return route_type @@ -1331,6 +1347,13 @@ def get_bedrock_chat_config(model: str): bedrock_invoke_provider: Final = BaseAWSLLM.get_bedrock_invoke_provider(model=model) base_model: Final = BedrockModelInfo.get_base_model(model) + from litellm.utils import model_supports_native_endpoint + + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/chat/completions", model, LlmProviders.BEDROCK + ): + return litellm.AmazonBedrockOpenAIChatCompletionsConfig() + # Handle explicit routes first if bedrock_route == "claude_platform": return litellm.BedrockClaudePlatformConfig() @@ -1357,11 +1380,6 @@ def get_bedrock_chat_config(model: str): return AmazonMantleConfig() - from litellm.utils import model_supports_native_endpoint - - if model_supports_native_endpoint("/v1/chat/completions", model, LlmProviders.BEDROCK): - return litellm.AmazonBedrockOpenAIChatCompletionsConfig() - # Handle provider-specific configs if bedrock_invoke_provider == "amazon": return litellm.AmazonTitanConfig() diff --git a/litellm/llms/bedrock/messages/native_transformation.py b/litellm/llms/bedrock/messages/native_transformation.py index 5571013d193..ca481851ced 100644 --- a/litellm/llms/bedrock/messages/native_transformation.py +++ b/litellm/llms/bedrock/messages/native_transformation.py @@ -67,11 +67,15 @@ class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig): api_key: str | None = None, api_base: str | None = None, ) -> tuple[dict, str | None]: # mutable-ok: provider interface + content_type: Final = next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ) return ( { # mutable-ok: provider interface - **headers, + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, "anthropic-version": headers.get("anthropic-version", DEFAULT_ANTHROPIC_API_VERSION), - "content-type": headers.get("content-type", "application/json"), + "Content-Type": content_type, }, api_base, ) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index e2a26245a29..8fc065fcde7 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,8 +52,11 @@ class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM): self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None ) -> dict: # mutable-ok: provider interface return { - **headers, - "Content-Type": headers.get("Content-Type", "application/json"), + **{key: value for key, value in headers.items() if key.lower() != "content-type"}, + "Content-Type": next( + (value for key, value in headers.items() if key.lower() == "content-type"), + "application/json", + ), } # mutable-ok: provider interface def sign_request( diff --git a/litellm/utils.py b/litellm/utils.py index 013d0393349..ac62b6fbadf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4489,7 +4489,17 @@ def get_optional_params( BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo") bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) bedrock_base_model: Final = BedrockModelInfo.get_base_model(model) - if bedrock_route == "converse" or bedrock_route == "converse_like": + if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint( + "/v1/chat/completions", model, LlmProviders.BEDROCK + ): + native_provider_config: Final = provider_config or litellm.AmazonBedrockOpenAIChatCompletionsConfig() + optional_params = native_provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + ) + elif bedrock_route == "converse" or bedrock_route == "converse_like": optional_params = litellm.AmazonConverseConfig().map_openai_params( model=model, non_default_params=non_default_params, @@ -8066,7 +8076,7 @@ def _get_model_cost_entry_for_provider_config( model: str, provider: LlmProviders, ) -> dict[str, Any]: - candidate_keys: Final = (model, f"{provider.value}/{model}") + candidate_keys: Final = (f"{provider.value}/{model}", model) for model_key in candidate_keys: model_info = litellm.model_cost.get(model_key) if model_info is not None: diff --git a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py index 571b62ed361..117179cebe4 100644 --- a/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py +++ b/tests/test_litellm/llms/bedrock/test_native_endpoint_forwarding.py @@ -106,6 +106,20 @@ def test_bedrock_chat_route_prefers_explicit_converse(monkeypatch): assert type(get_bedrock_chat_config("converse/native-model")).__name__ == "AmazonConverseConfig" +def test_bedrock_converse_catalog_model_prefers_declared_native_chat(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + + assert isinstance( + get_bedrock_chat_config("anthropic.claude-sonnet-4-6"), + AmazonBedrockOpenAIChatCompletionsConfig, + ) + assert type(get_bedrock_chat_config("converse/anthropic.claude-sonnet-4-6")).__name__ == "AmazonConverseConfig" + + def test_native_chat_url_and_bearer_auth(monkeypatch): config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture @@ -114,13 +128,21 @@ def test_native_chat_url_and_bearer_auth(monkeypatch): ) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") - headers, body = config.sign_request( + validated_headers = config.validate_environment( + {"content-type": "application/json"}, # mutable-ok: provider interface + "native-model", + [], # mutable-ok: provider interface {}, # mutable-ok: provider interface + {}, # mutable-ok: provider interface + ) + headers, body = config.sign_request( + validated_headers, {"aws_region_name": "us-west-2"}, # mutable-ok: provider interface {"model": "native-model"}, # mutable-ok: provider interface "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions", ) assert headers["Authorization"] == "Bearer token" + assert sum(key.lower() == "content-type" for key in headers) == 1 assert body is not None @@ -176,9 +198,25 @@ def test_native_messages_preserves_body_and_urls(monkeypatch): "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages", ) assert headers["Authorization"] == "Bearer token" + assert sum(key.lower() == "content-type" for key in headers) == 1 assert signed_body is not None +def test_native_messages_deduplicates_content_type_header(monkeypatch): + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token") + config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture + headers, _ = config.validate_anthropic_messages_environment( + headers={"content-type": "application/json"}, # mutable-ok: provider interface + model="native-model", + messages=[], # mutable-ok: provider interface + optional_params={}, # mutable-ok: provider interface + litellm_params={}, # mutable-ok: provider interface + ) + + assert sum(key.lower() == "content-type" for key in headers) == 1 + assert headers["Content-Type"] == "application/json" + + def test_native_messages_selection_is_not_cached(monkeypatch): monkeypatch.setitem( litellm.model_cost, @@ -214,6 +252,12 @@ def test_native_responses_selection_and_url(monkeypatch): assert config.get_complete_url(None, {"aws_region_name": "us-west-2"}).endswith( # mutable-ok: provider interface "/openai/v1/responses" ) + validated_headers = config.validate_environment( + {"content-type": "application/json"}, # mutable-ok: provider interface + "native-model", + {}, # mutable-ok: provider interface + ) + assert sum(key.lower() == "content-type" for key in validated_headers) == 1 monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface assert ProviderConfigManager.get_provider_responses_api_config( @@ -266,6 +310,31 @@ def test_completion_forwards_native_chat_body(monkeypatch): ] +def test_completion_forwards_declared_native_chat_for_converse_catalog_model(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "bedrock/anthropic.claude-sonnet-4-6", + {"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface + ) + requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture + + def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility + requests.append((url, json.loads(data))) + return _chat_response(url) + + with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post + ): + litellm.completion( + model="bedrock/anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface + api_base="https://bedrock-runtime.us-west-2.amazonaws.com", + api_key="token", + ) + + assert requests[0][0].endswith("/openai/v1/chat/completions") + + @pytest.mark.asyncio async def test_anthropic_messages_forwards_native_body(monkeypatch): monkeypatch.setitem(