diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index ec5b942ec1b..26986aab586 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -4,6 +4,9 @@ from typing import Any, Optional, Union import httpx import litellm +from litellm.anthropic_beta_headers_manager import ( + update_headers_with_filtered_beta, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -13,11 +16,9 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper -from litellm.anthropic_beta_headers_manager import ( - update_headers_with_filtered_beta, - ) + from ..base_aws_llm import BaseAWSLLM, Credentials -from ..common_utils import BedrockError +from ..common_utils import BedrockError, _get_all_bedrock_regions from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call @@ -279,11 +280,22 @@ class BedrockConverseLLM(BaseAWSLLM): if _stripped.startswith(rp): _stripped = _stripped[len(rp):] break + # Strip embedded region prefix (e.g. "bedrock/us-east-1/model" -> "model") + # and capture it so it can be used as aws_region_name below. + _region_from_model: Optional[str] = None + _potential_region = _stripped.split("/", 1)[0] + if _potential_region in _get_all_bedrock_regions() and "/" in _stripped: + _region_from_model = _potential_region + _stripped = _stripped.split("/", 1)[1] + _model_for_id = _stripped for _nova_prefix in ["nova-2/", "nova/"]: if _stripped.startswith(_nova_prefix): _model_for_id = _model_for_id.replace(_nova_prefix, "", 1) break modelId = self.encode_model_id(model_id=_model_for_id) + # Inject region extracted from model path so _get_aws_region_name picks it up + if _region_from_model is not None and "aws_region_name" not in optional_params: + optional_params["aws_region_name"] = _region_from_model fake_stream = litellm.AmazonConverseConfig().should_fake_stream( fake_stream=fake_stream, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f52288ea72a..cbd64a178b8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16289,7 +16289,7 @@ "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, - "litellm_provider": "vertex_ai-language-models", + "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, "supports_reasoning": false, diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index 9d8371c04da..f207c1d272a 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -1,12 +1,14 @@ import os import sys +import pytest + from litellm.llms.bedrock.chat import BedrockConverseLLM +from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path -import litellm def test_encode_model_id_with_inference_profile(): @@ -18,3 +20,116 @@ def test_encode_model_id_with_inference_profile(): bedrock_converse_llm = BedrockConverseLLM() returned_model = bedrock_converse_llm.encode_model_id(test_model) assert expected_model == returned_model + + +class TestBedrockRegionInModelPath: + """ + Tests for region extraction from bedrock/{region}/{model} path format. + + When a user passes model="bedrock/ap-northeast-1/moonshotai.kimi-k2.5", + get_llm_provider strips "bedrock/" and passes "ap-northeast-1/moonshotai.kimi-k2.5" + to the converse handler. The handler must: + 1. Strip the region from modelId (so AWS gets "moonshotai.kimi-k2.5", not "ap-northeast-1%2Fmoonshotai.kimi-k2.5") + 2. Use the extracted region as aws_region_name for the API call + """ + + @pytest.mark.parametrize( + "model,expected_model_id,expected_region", + [ + # Region embedded in path — both modelId and region must be extracted + ( + "ap-northeast-1/moonshotai.kimi-k2.5", + "moonshotai.kimi-k2.5", + "ap-northeast-1", + ), + ( + "us-east-1/moonshotai.kimi-k2.5", + "moonshotai.kimi-k2.5", + "us-east-1", + ), + ( + "us-west-2/anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-3-5-sonnet-20241022-v2%3A0", + "us-west-2", + ), + # No region in path — modelId unchanged, no region injected + ( + "moonshotai.kimi-k2.5", + "moonshotai.kimi-k2.5", + None, + ), + # Cross-region inference prefix (us., eu., ap.) — not a region path segment + ( + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "us.anthropic.claude-3-5-sonnet-20241022-v2%3A0", + None, + ), + ], + ) + def test_region_and_model_id_extraction( + self, model, expected_model_id, expected_region + ): + """ + Verify that completion() correctly extracts both modelId and aws_region_name + from the bedrock/{region}/{model} path format. + """ + bedrock_converse_llm = BedrockConverseLLM() + optional_params: dict = {} + + # Simulate the modelId + region extraction logic from completion() + _model_for_id = model + _stripped = _model_for_id + for rp in ["bedrock/converse/", "bedrock/", "converse/"]: + if _stripped.startswith(rp): + _stripped = _stripped[len(rp):] + break + + _region_from_model = None + _potential_region = _stripped.split("/", 1)[0] + if _potential_region in _get_all_bedrock_regions() and "/" in _stripped: + _region_from_model = _potential_region + _stripped = _stripped.split("/", 1)[1] + _model_for_id = _stripped + + for _nova_prefix in ["nova-2/", "nova/"]: + if _stripped.startswith(_nova_prefix): + _model_for_id = _model_for_id.replace(_nova_prefix, "", 1) + break + + model_id = bedrock_converse_llm.encode_model_id(model_id=_model_for_id) + if _region_from_model is not None and "aws_region_name" not in optional_params: + optional_params["aws_region_name"] = _region_from_model + + assert model_id == expected_model_id, ( + f"modelId mismatch for {model!r}: got {model_id!r}, expected {expected_model_id!r}" + ) + assert optional_params.get("aws_region_name") == expected_region, ( + f"region mismatch for {model!r}: got {optional_params.get('aws_region_name')!r}, expected {expected_region!r}" + ) + + def test_explicit_aws_region_name_not_overridden(self): + """ + If aws_region_name is already set in optional_params, the region in the + model path must NOT override it. + """ + bedrock_converse_llm = BedrockConverseLLM() + optional_params = {"aws_region_name": "eu-west-1"} + model = "ap-northeast-1/moonshotai.kimi-k2.5" + + _model_for_id = model + _stripped = model + _region_from_model = None + _potential_region = _stripped.split("/", 1)[0] + if _potential_region in _get_all_bedrock_regions() and "/" in _stripped: + _region_from_model = _potential_region + _stripped = _stripped.split("/", 1)[1] + _model_for_id = _stripped + + model_id = bedrock_converse_llm.encode_model_id(model_id=_model_for_id) + if _region_from_model is not None and "aws_region_name" not in optional_params: + optional_params["aws_region_name"] = _region_from_model + + # modelId is still correctly stripped + assert model_id == "moonshotai.kimi-k2.5" + # explicitly set region is preserved + assert optional_params["aws_region_name"] == "eu-west-1"