mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix: honor native chat gate for converse models and dedupe content-type
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
21a72defa5
commit
28079dd187
6 changed files with 149 additions and 39 deletions
|
|
@ -69,7 +69,13 @@ class AmazonBedrockOpenAIChatCompletionsConfig(OpenAIGPTConfig, BaseAWSLLM):
|
|||
) -> dict: # mutable-ok: provider interface
|
||||
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
|
||||
return merge_bedrock_invoke_headers(
|
||||
headers,
|
||||
{
|
||||
**{key: value for key, value in headers.items() if key.lower() != "content-type"},
|
||||
"Content-Type": next(
|
||||
(value for key, value in headers.items() if key.lower() == "content-type"),
|
||||
"application/json",
|
||||
),
|
||||
},
|
||||
(),
|
||||
metadata_headers,
|
||||
owned_names,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,43 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
|
||||
_BEDROCK_ROUTE_PREFIXES: Final[tuple[str, ...]] = (
|
||||
"invoke/",
|
||||
"claude_platform/",
|
||||
"converse_like/",
|
||||
"converse/",
|
||||
"agent/",
|
||||
"agentcore/",
|
||||
"async_invoke/",
|
||||
"openai/",
|
||||
"mantle/",
|
||||
)
|
||||
_BEDROCK_ROUTE_TYPES: Final[
|
||||
tuple[
|
||||
Literal[
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"converse",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
],
|
||||
...,
|
||||
]
|
||||
] = (
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"converse",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
)
|
||||
|
||||
|
||||
def error_response_text(response: httpx.Response) -> str:
|
||||
|
|
@ -1090,6 +1127,10 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
"""Wrapper for standalone function. See get_bedrock_base_model()."""
|
||||
return get_bedrock_base_model(model)
|
||||
|
||||
@staticmethod
|
||||
def has_explicit_route(model: str) -> bool:
|
||||
return any(BedrockModelInfo._model_has_route_prefix(model, prefix) for prefix in _BEDROCK_ROUTE_PREFIXES)
|
||||
|
||||
@staticmethod
|
||||
def _supported_cross_region_inference_region() -> list[str]:
|
||||
"""Wrapper for standalone function. See get_bedrock_cross_region_inference_regions()."""
|
||||
|
|
@ -1112,36 +1153,11 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
"""
|
||||
Get the bedrock route for the given model.
|
||||
"""
|
||||
route_mappings: dict[
|
||||
str,
|
||||
Literal[
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"converse",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
],
|
||||
] = {
|
||||
"invoke/": "invoke",
|
||||
"claude_platform/": "claude_platform",
|
||||
"converse_like/": "converse_like",
|
||||
"converse/": "converse",
|
||||
"agent/": "agent",
|
||||
"agentcore/": "agentcore",
|
||||
"async_invoke/": "async_invoke",
|
||||
"openai/": "openai",
|
||||
"mantle/": "mantle",
|
||||
}
|
||||
|
||||
# Check explicit routes first. Match each prefix only as a leading path
|
||||
# segment so the `bedrock_mantle/` provider prefix is never mistaken for
|
||||
# the `mantle/` invoke route (which would mangle
|
||||
# `bedrock_mantle/openai.gpt-5.5` into `bedrock_openai.gpt-5.5`).
|
||||
for prefix, route_type in route_mappings.items():
|
||||
for prefix, route_type in zip(_BEDROCK_ROUTE_PREFIXES, _BEDROCK_ROUTE_TYPES, strict=True):
|
||||
if BedrockModelInfo._model_has_route_prefix(model, prefix):
|
||||
return route_type
|
||||
|
||||
|
|
@ -1331,6 +1347,13 @@ def get_bedrock_chat_config(model: str):
|
|||
bedrock_invoke_provider: Final = BaseAWSLLM.get_bedrock_invoke_provider(model=model)
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
|
||||
from litellm.utils import model_supports_native_endpoint
|
||||
|
||||
if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint(
|
||||
"/v1/chat/completions", model, LlmProviders.BEDROCK
|
||||
):
|
||||
return litellm.AmazonBedrockOpenAIChatCompletionsConfig()
|
||||
|
||||
# Handle explicit routes first
|
||||
if bedrock_route == "claude_platform":
|
||||
return litellm.BedrockClaudePlatformConfig()
|
||||
|
|
@ -1357,11 +1380,6 @@ def get_bedrock_chat_config(model: str):
|
|||
|
||||
return AmazonMantleConfig()
|
||||
|
||||
from litellm.utils import model_supports_native_endpoint
|
||||
|
||||
if model_supports_native_endpoint("/v1/chat/completions", model, LlmProviders.BEDROCK):
|
||||
return litellm.AmazonBedrockOpenAIChatCompletionsConfig()
|
||||
|
||||
# Handle provider-specific configs
|
||||
if bedrock_invoke_provider == "amazon":
|
||||
return litellm.AmazonTitanConfig()
|
||||
|
|
|
|||
|
|
@ -67,11 +67,15 @@ class AmazonBedrockNativeMessagesConfig(BaseAWSLLM, AnthropicMessagesConfig):
|
|||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[dict, str | None]: # mutable-ok: provider interface
|
||||
content_type: Final = next(
|
||||
(value for key, value in headers.items() if key.lower() == "content-type"),
|
||||
"application/json",
|
||||
)
|
||||
return (
|
||||
{ # mutable-ok: provider interface
|
||||
**headers,
|
||||
**{key: value for key, value in headers.items() if key.lower() != "content-type"},
|
||||
"anthropic-version": headers.get("anthropic-version", DEFAULT_ANTHROPIC_API_VERSION),
|
||||
"content-type": headers.get("content-type", "application/json"),
|
||||
"Content-Type": content_type,
|
||||
},
|
||||
api_base,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -52,8 +52,11 @@ class AmazonBedrockResponsesAPIConfig(OpenAIResponsesAPIConfig, BaseAWSLLM):
|
|||
self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None
|
||||
) -> dict: # mutable-ok: provider interface
|
||||
return {
|
||||
**headers,
|
||||
"Content-Type": headers.get("Content-Type", "application/json"),
|
||||
**{key: value for key, value in headers.items() if key.lower() != "content-type"},
|
||||
"Content-Type": next(
|
||||
(value for key, value in headers.items() if key.lower() == "content-type"),
|
||||
"application/json",
|
||||
),
|
||||
} # mutable-ok: provider interface
|
||||
|
||||
def sign_request(
|
||||
|
|
|
|||
|
|
@ -4489,7 +4489,17 @@ def get_optional_params(
|
|||
BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo")
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
|
||||
bedrock_base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if bedrock_route == "converse" or bedrock_route == "converse_like":
|
||||
if not BedrockModelInfo.has_explicit_route(model) and model_supports_native_endpoint(
|
||||
"/v1/chat/completions", model, LlmProviders.BEDROCK
|
||||
):
|
||||
native_provider_config: Final = provider_config or litellm.AmazonBedrockOpenAIChatCompletionsConfig()
|
||||
optional_params = native_provider_config.map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif bedrock_route == "converse" or bedrock_route == "converse_like":
|
||||
optional_params = litellm.AmazonConverseConfig().map_openai_params(
|
||||
model=model,
|
||||
non_default_params=non_default_params,
|
||||
|
|
@ -8066,7 +8076,7 @@ def _get_model_cost_entry_for_provider_config(
|
|||
model: str,
|
||||
provider: LlmProviders,
|
||||
) -> dict[str, Any]:
|
||||
candidate_keys: Final = (model, f"{provider.value}/{model}")
|
||||
candidate_keys: Final = (f"{provider.value}/{model}", model)
|
||||
for model_key in candidate_keys:
|
||||
model_info = litellm.model_cost.get(model_key)
|
||||
if model_info is not None:
|
||||
|
|
|
|||
|
|
@ -106,6 +106,20 @@ def test_bedrock_chat_route_prefers_explicit_converse(monkeypatch):
|
|||
assert type(get_bedrock_chat_config("converse/native-model")).__name__ == "AmazonConverseConfig"
|
||||
|
||||
|
||||
def test_bedrock_converse_catalog_model_prefers_declared_native_chat(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/anthropic.claude-sonnet-4-6",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
get_bedrock_chat_config("anthropic.claude-sonnet-4-6"),
|
||||
AmazonBedrockOpenAIChatCompletionsConfig,
|
||||
)
|
||||
assert type(get_bedrock_chat_config("converse/anthropic.claude-sonnet-4-6")).__name__ == "AmazonConverseConfig"
|
||||
|
||||
|
||||
def test_native_chat_url_and_bearer_auth(monkeypatch):
|
||||
config = AmazonBedrockOpenAIChatCompletionsConfig() # rebind-ok: test capture
|
||||
|
||||
|
|
@ -114,13 +128,21 @@ def test_native_chat_url_and_bearer_auth(monkeypatch):
|
|||
) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
|
||||
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token")
|
||||
headers, body = config.sign_request(
|
||||
validated_headers = config.validate_environment(
|
||||
{"content-type": "application/json"}, # mutable-ok: provider interface
|
||||
"native-model",
|
||||
[], # mutable-ok: provider interface
|
||||
{}, # mutable-ok: provider interface
|
||||
{}, # mutable-ok: provider interface
|
||||
)
|
||||
headers, body = config.sign_request(
|
||||
validated_headers,
|
||||
{"aws_region_name": "us-west-2"}, # mutable-ok: provider interface
|
||||
{"model": "native-model"}, # mutable-ok: provider interface
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions",
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer token"
|
||||
assert sum(key.lower() == "content-type" for key in headers) == 1
|
||||
assert body is not None
|
||||
|
||||
|
||||
|
|
@ -176,9 +198,25 @@ def test_native_messages_preserves_body_and_urls(monkeypatch):
|
|||
"https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages",
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer token"
|
||||
assert sum(key.lower() == "content-type" for key in headers) == 1
|
||||
assert signed_body is not None
|
||||
|
||||
|
||||
def test_native_messages_deduplicates_content_type_header(monkeypatch):
|
||||
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "token")
|
||||
config = AmazonBedrockNativeMessagesConfig() # rebind-ok: test capture
|
||||
headers, _ = config.validate_anthropic_messages_environment(
|
||||
headers={"content-type": "application/json"}, # mutable-ok: provider interface
|
||||
model="native-model",
|
||||
messages=[], # mutable-ok: provider interface
|
||||
optional_params={}, # mutable-ok: provider interface
|
||||
litellm_params={}, # mutable-ok: provider interface
|
||||
)
|
||||
|
||||
assert sum(key.lower() == "content-type" for key in headers) == 1
|
||||
assert headers["Content-Type"] == "application/json"
|
||||
|
||||
|
||||
def test_native_messages_selection_is_not_cached(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
|
|
@ -214,6 +252,12 @@ def test_native_responses_selection_and_url(monkeypatch):
|
|||
assert config.get_complete_url(None, {"aws_region_name": "us-west-2"}).endswith( # mutable-ok: provider interface
|
||||
"/openai/v1/responses"
|
||||
)
|
||||
validated_headers = config.validate_environment(
|
||||
{"content-type": "application/json"}, # mutable-ok: provider interface
|
||||
"native-model",
|
||||
{}, # mutable-ok: provider interface
|
||||
)
|
||||
assert sum(key.lower() == "content-type" for key in validated_headers) == 1
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, "bedrock/native-model", {}) # mutable-ok: provider interface
|
||||
assert ProviderConfigManager.get_provider_responses_api_config(
|
||||
|
|
@ -266,6 +310,31 @@ def test_completion_forwards_native_chat_body(monkeypatch):
|
|||
]
|
||||
|
||||
|
||||
def test_completion_forwards_declared_native_chat_for_converse_catalog_model(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"bedrock/anthropic.claude-sonnet-4-6",
|
||||
{"supported_endpoints": ["/v1/chat/completions"]}, # mutable-ok: provider interface
|
||||
)
|
||||
requests: list[tuple[str, dict]] = [] # mutable-ok: provider interface # rebind-ok: test capture
|
||||
|
||||
def post(self, url, data=None, headers=None, **kwargs): # kwargs-ok: HTTP handler compatibility
|
||||
requests.append((url, json.loads(data)))
|
||||
return _chat_response(url)
|
||||
|
||||
with patch( # test-quality-ok: captures the serialized provider request at the HTTP boundary
|
||||
"litellm.llms.custom_httpx.http_handler.HTTPHandler.post", post
|
||||
):
|
||||
litellm.completion(
|
||||
model="bedrock/anthropic.claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "hello"}], # mutable-ok: provider interface
|
||||
api_base="https://bedrock-runtime.us-west-2.amazonaws.com",
|
||||
api_key="token",
|
||||
)
|
||||
|
||||
assert requests[0][0].endswith("/openai/v1/chat/completions")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_forwards_native_body(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue