Merge pull request #22546 from BerriAI/litellm_bedrock_region_in_model_path

fix(bedrock): extract region and model ID from bedrock/{region}/{model} path format

litellm/llms/bedrock/chat/converse_handler.py

@@ -4,6 +4,9 @@ from typing import Any, Optional, Union
 import httpx
 
 import litellm
+from litellm.anthropic_beta_headers_manager import (
+    update_headers_with_filtered_beta,
+)
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -13,11 +16,9 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.types.utils import ModelResponse
 from litellm.utils import CustomStreamWrapper
-from litellm.anthropic_beta_headers_manager import (
-            update_headers_with_filtered_beta,
-        )
+
 from ..base_aws_llm import BaseAWSLLM, Credentials
-from ..common_utils import BedrockError
+from ..common_utils import BedrockError, _get_all_bedrock_regions
 from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
 
 
@@ -279,11 +280,22 @@ class BedrockConverseLLM(BaseAWSLLM):
                 if _stripped.startswith(rp):
                     _stripped = _stripped[len(rp):]
                     break
+            # Strip embedded region prefix (e.g. "bedrock/us-east-1/model" -> "model")
+            # and capture it so it can be used as aws_region_name below.
+            _region_from_model: Optional[str] = None
+            _potential_region = _stripped.split("/", 1)[0]
+            if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
+                _region_from_model = _potential_region
+                _stripped = _stripped.split("/", 1)[1]
+                _model_for_id = _stripped
             for _nova_prefix in ["nova-2/", "nova/"]:
                 if _stripped.startswith(_nova_prefix):
                     _model_for_id = _model_for_id.replace(_nova_prefix, "", 1)
                     break
             modelId = self.encode_model_id(model_id=_model_for_id)
+            # Inject region extracted from model path so _get_aws_region_name picks it up
+            if _region_from_model is not None and "aws_region_name" not in optional_params:
+                optional_params["aws_region_name"] = _region_from_model
 
         fake_stream = litellm.AmazonConverseConfig().should_fake_stream(
             fake_stream=fake_stream,

litellm/model_prices_and_context_window_backup.json

@@ -16289,7 +16289,7 @@
         "cache_read_input_token_cost": 3e-08,
         "input_cost_per_audio_token": 1e-06,
         "input_cost_per_token": 3e-07,
-        "litellm_provider": "vertex_ai-language-models",
+        "litellm_provider": "gemini",
         "max_audio_length_hours": 8.4,
         "max_audio_per_prompt": 1,
         "supports_reasoning": false,

tests/test_litellm/llms/chat/test_converse_handler.py

@@ -1,12 +1,14 @@
 import os
 import sys
 
+import pytest
+
 from litellm.llms.bedrock.chat import BedrockConverseLLM
+from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions
 
 sys.path.insert(
     0, os.path.abspath("../../../../..")
 )  # Adds the parent directory to the system path
-import litellm
 
 
 def test_encode_model_id_with_inference_profile():
@@ -18,3 +20,116 @@ def test_encode_model_id_with_inference_profile():
     bedrock_converse_llm = BedrockConverseLLM()
     returned_model = bedrock_converse_llm.encode_model_id(test_model)
     assert expected_model == returned_model
+
+
+class TestBedrockRegionInModelPath:
+    """
+    Tests for region extraction from bedrock/{region}/{model} path format.
+
+    When a user passes model="bedrock/ap-northeast-1/moonshotai.kimi-k2.5",
+    get_llm_provider strips "bedrock/" and passes "ap-northeast-1/moonshotai.kimi-k2.5"
+    to the converse handler. The handler must:
+    1. Strip the region from modelId (so AWS gets "moonshotai.kimi-k2.5", not "ap-northeast-1%2Fmoonshotai.kimi-k2.5")
+    2. Use the extracted region as aws_region_name for the API call
+    """
+
+    @pytest.mark.parametrize(
+        "model,expected_model_id,expected_region",
+        [
+            # Region embedded in path — both modelId and region must be extracted
+            (
+                "ap-northeast-1/moonshotai.kimi-k2.5",
+                "moonshotai.kimi-k2.5",
+                "ap-northeast-1",
+            ),
+            (
+                "us-east-1/moonshotai.kimi-k2.5",
+                "moonshotai.kimi-k2.5",
+                "us-east-1",
+            ),
+            (
+                "us-west-2/anthropic.claude-3-5-sonnet-20241022-v2:0",
+                "anthropic.claude-3-5-sonnet-20241022-v2%3A0",
+                "us-west-2",
+            ),
+            # No region in path — modelId unchanged, no region injected
+            (
+                "moonshotai.kimi-k2.5",
+                "moonshotai.kimi-k2.5",
+                None,
+            ),
+            # Cross-region inference prefix (us., eu., ap.) — not a region path segment
+            (
+                "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+                "us.anthropic.claude-3-5-sonnet-20241022-v2%3A0",
+                None,
+            ),
+        ],
+    )
+    def test_region_and_model_id_extraction(
+        self, model, expected_model_id, expected_region
+    ):
+        """
+        Verify that completion() correctly extracts both modelId and aws_region_name
+        from the bedrock/{region}/{model} path format.
+        """
+        bedrock_converse_llm = BedrockConverseLLM()
+        optional_params: dict = {}
+
+        # Simulate the modelId + region extraction logic from completion()
+        _model_for_id = model
+        _stripped = _model_for_id
+        for rp in ["bedrock/converse/", "bedrock/", "converse/"]:
+            if _stripped.startswith(rp):
+                _stripped = _stripped[len(rp):]
+                break
+
+        _region_from_model = None
+        _potential_region = _stripped.split("/", 1)[0]
+        if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
+            _region_from_model = _potential_region
+            _stripped = _stripped.split("/", 1)[1]
+            _model_for_id = _stripped
+
+        for _nova_prefix in ["nova-2/", "nova/"]:
+            if _stripped.startswith(_nova_prefix):
+                _model_for_id = _model_for_id.replace(_nova_prefix, "", 1)
+                break
+
+        model_id = bedrock_converse_llm.encode_model_id(model_id=_model_for_id)
+        if _region_from_model is not None and "aws_region_name" not in optional_params:
+            optional_params["aws_region_name"] = _region_from_model
+
+        assert model_id == expected_model_id, (
+            f"modelId mismatch for {model!r}: got {model_id!r}, expected {expected_model_id!r}"
+        )
+        assert optional_params.get("aws_region_name") == expected_region, (
+            f"region mismatch for {model!r}: got {optional_params.get('aws_region_name')!r}, expected {expected_region!r}"
+        )
+
+    def test_explicit_aws_region_name_not_overridden(self):
+        """
+        If aws_region_name is already set in optional_params, the region in the
+        model path must NOT override it.
+        """
+        bedrock_converse_llm = BedrockConverseLLM()
+        optional_params = {"aws_region_name": "eu-west-1"}
+        model = "ap-northeast-1/moonshotai.kimi-k2.5"
+
+        _model_for_id = model
+        _stripped = model
+        _region_from_model = None
+        _potential_region = _stripped.split("/", 1)[0]
+        if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
+            _region_from_model = _potential_region
+            _stripped = _stripped.split("/", 1)[1]
+            _model_for_id = _stripped
+
+        model_id = bedrock_converse_llm.encode_model_id(model_id=_model_for_id)
+        if _region_from_model is not None and "aws_region_name" not in optional_params:
+            optional_params["aws_region_name"] = _region_from_model
+
+        # modelId is still correctly stripped
+        assert model_id == "moonshotai.kimi-k2.5"
+        # explicitly set region is preserved
+        assert optional_params["aws_region_name"] == "eu-west-1"