From 8e8511a2a393f82faceb314b82c9135ea5603b00 Mon Sep 17 00:00:00 2001 From: ryanh-ai <3118399+ryanh-ai@users.noreply.github.com> Date: Tue, 17 Feb 2026 23:00:37 -0800 Subject: [PATCH] feat(bedrock): support nova/ and nova-2/ spec prefixes for custom imported models (#21359) Add routing prefixes bedrock/nova/ and bedrock/nova-2/ so LiteLLM can identify the base model family for custom/imported Nova models and enable the correct supported params (tools, web_search, reasoning_effort). Changes: - Route nova/ and nova-2/ prefixed models to converse API - Strip spec prefix before sending ARN to Bedrock - Return sentinel base models (amazon.nova-custom, amazon.nova-2-custom) so downstream Nova checks work - Recognize nova-2/ prefix in _is_nova_2_model() for reasoning support - Handle nova/nova-2 in get_bedrock_model_id() for proper ARN encoding - Add unit tests for all new behavior --- litellm/llms/bedrock/base_aws_llm.py | 8 ++ litellm/llms/bedrock/chat/converse_handler.py | 13 ++- .../bedrock/chat/converse_transformation.py | 6 +- litellm/llms/bedrock/common_utils.py | 20 +++- .../llms/bedrock/test_nova_imported_models.py | 92 +++++++++++++++++++ 5 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 tests/litellm/llms/bedrock/test_nova_imported_models.py diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 304c707fa0b..dfaddb3c2b1 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -384,6 +384,14 @@ class BaseAWSLLM: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="moonshot" ) + elif "nova-2/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="nova-2" + ) + elif "nova/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="nova" + ) return model_id @staticmethod diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 25af852e09c..60a93b169c8 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -272,7 +272,18 @@ class BedrockConverseLLM(BaseAWSLLM): if unencoded_model_id is not None: modelId = self.encode_model_id(model_id=unencoded_model_id) else: - modelId = self.encode_model_id(model_id=model) + # Strip nova spec prefixes before encoding model ID for API URL + _model_for_id = model + _stripped = _model_for_id + for rp in ["bedrock/converse/", "bedrock/", "converse/"]: + if _stripped.startswith(rp): + _stripped = _stripped[len(rp):] + break + for _nova_prefix in ["nova-2/", "nova/"]: + if _stripped.startswith(_nova_prefix): + _model_for_id = _model_for_id.replace(_nova_prefix, "", 1) + break + modelId = self.encode_model_id(model_id=_model_for_id) fake_stream = litellm.AmazonConverseConfig().should_fake_stream( fake_stream=fake_stream, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4e3be8edc65..b2aece741cd 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -318,7 +318,8 @@ class AmazonConverseConfig(BaseConfig): break # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.) - return model_without_region.startswith("amazon.nova-2-") + # Also check for nova-2/ spec prefix for imported models + return model_without_region.startswith("amazon.nova-2-") or model_without_region.startswith("nova-2/") def _map_web_search_options( self, web_search_options: dict, model: str @@ -490,6 +491,9 @@ class AmazonConverseConfig(BaseConfig): supported_params.append("tool_choice") supported_params.append("thinking") supported_params.append("reasoning_effort") + # For nova imported models, also add web_search_options + if "nova" in model.lower(): + supported_params.append("web_search_options") return supported_params ## Filter out 'cross-region' from model name diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4c87f6fa994..8d40e349155 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -404,7 +404,7 @@ def extract_model_name_from_bedrock_arn(model: str) -> str: def strip_bedrock_routing_prefix(model: str) -> str: """Strip LiteLLM routing prefixes from model name.""" - for prefix in ["bedrock/", "converse/", "invoke/", "openai/"]: + for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "nova-2/", "nova/"]: if model.startswith(prefix): model = model.split("/", 1)[1] return model @@ -427,7 +427,20 @@ def get_bedrock_base_model(model: str) -> str: - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" - "bedrock/converse/model" -> "model" - "anthropic.claude-3-5-sonnet-20241022-v2:0:51k" -> "anthropic.claude-3-5-sonnet-20241022-v2:0" + - "bedrock/nova-2/arn:aws:..." -> "amazon.nova-2-custom" + - "bedrock/nova/arn:aws:..." -> "amazon.nova-custom" """ + # Detect nova spec prefixes before stripping them + stripped = model + for rp in ["bedrock/converse/", "bedrock/", "converse/"]: + if stripped.startswith(rp): + stripped = stripped[len(rp):] + break + if stripped.startswith("nova-2/"): + return "amazon.nova-2-custom" + elif stripped.startswith("nova/"): + return "amazon.nova-custom" + model = strip_bedrock_routing_prefix(model) model = extract_model_name_from_bedrock_arn(model) model = strip_bedrock_throughput_suffix(model) @@ -594,6 +607,11 @@ class BedrockModelInfo(BaseLLMModelInfo): if prefix in model: return route_type + # Check for nova spec prefixes (nova/ and nova-2/) + _model_after_bedrock = model.replace("bedrock/", "", 1) + if _model_after_bedrock.startswith("nova-2/") or _model_after_bedrock.startswith("nova/"): + return "converse" + base_model = BedrockModelInfo.get_base_model(model) alt_model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model) if ( diff --git a/tests/litellm/llms/bedrock/test_nova_imported_models.py b/tests/litellm/llms/bedrock/test_nova_imported_models.py new file mode 100644 index 00000000000..e3677aaf9e6 --- /dev/null +++ b/tests/litellm/llms/bedrock/test_nova_imported_models.py @@ -0,0 +1,92 @@ +""" +Tests for Nova imported/custom model support via spec prefixes (nova/, nova-2/). +""" + +import pytest + +from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo, + get_bedrock_base_model, + strip_bedrock_routing_prefix, +) +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + + +NOVA_ARN = "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/a1b2c3d4e5f6" +NOVA_MODEL = f"bedrock/nova/{NOVA_ARN}" +NOVA2_MODEL = f"bedrock/nova-2/{NOVA_ARN}" + + +class TestGetBedrockRoute: + def test_nova_prefix_routes_to_converse(self): + assert BedrockModelInfo.get_bedrock_route(NOVA_MODEL) == "converse" + + def test_nova2_prefix_routes_to_converse(self): + assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse" + + def test_plain_arn_routes_to_invoke(self): + # Without spec prefix, ARN doesn't match converse models + result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") + assert result == "invoke" + + +class TestGetBedrockBaseModel: + def test_nova_prefix_returns_sentinel(self): + assert get_bedrock_base_model(f"nova/{NOVA_ARN}") == "amazon.nova-custom" + + def test_nova2_prefix_returns_sentinel(self): + assert get_bedrock_base_model(f"nova-2/{NOVA_ARN}") == "amazon.nova-2-custom" + + def test_bedrock_nova_prefix_returns_sentinel(self): + assert get_bedrock_base_model(NOVA_MODEL) == "amazon.nova-custom" + + def test_bedrock_nova2_prefix_returns_sentinel(self): + assert get_bedrock_base_model(NOVA2_MODEL) == "amazon.nova-2-custom" + + +class TestStripBedrockRoutingPrefix: + def test_strips_nova_prefix(self): + result = strip_bedrock_routing_prefix(f"nova/{NOVA_ARN}") + assert result == NOVA_ARN + + def test_strips_nova2_prefix(self): + result = strip_bedrock_routing_prefix(f"nova-2/{NOVA_ARN}") + assert result == NOVA_ARN + + +class TestIsNova2Model: + def setup_method(self): + self.config = AmazonConverseConfig() + + def test_standard_nova2_model(self): + assert self.config._is_nova_2_model("amazon.nova-2-lite-v1:0") is True + + def test_nova2_imported_model(self): + assert self.config._is_nova_2_model(NOVA2_MODEL) is True + + def test_nova_imported_model_is_not_nova2(self): + assert self.config._is_nova_2_model(NOVA_MODEL) is False + + def test_plain_nova_model(self): + assert self.config._is_nova_2_model("amazon.nova-pro-v1:0") is False + + +class TestGetSupportedOpenaiParams: + def setup_method(self): + self.config = AmazonConverseConfig() + + def test_nova_imported_has_tools_and_web_search(self): + params = self.config.get_supported_openai_params(NOVA_MODEL) + assert "tools" in params + assert "tool_choice" in params + assert "web_search_options" in params + + def test_nova2_imported_has_reasoning_effort(self): + params = self.config.get_supported_openai_params(NOVA2_MODEL) + assert "reasoning_effort" in params + assert "web_search_options" in params + + def test_nova2_imported_has_tools(self): + params = self.config.get_supported_openai_params(NOVA2_MODEL) + assert "tools" in params + assert "tool_choice" in params