From 395b70c7622105c211f224002933478207c1369b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:13:21 -0700 Subject: [PATCH 1/6] fix(bedrock): send region-prefixed embedding model ids to that region without the prefix --- litellm/llms/bedrock/embed/embedding.py | 62 +++++++++------- .../bedrock/embed/test_bedrock_embedding.py | 71 +++++++++++++++++++ 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 58cfb6cb95e..762e9297b59 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -35,7 +35,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError +from ..common_utils import BedrockError, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -63,12 +63,18 @@ def _sign_get_request( return request.prepare() +def _without_region_path(model: str) -> str: + region, model_id = split_bedrock_region_path(model) + return model if region is None else model_id + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, + region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -76,15 +82,17 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, + region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, + region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) + aws_region_name = optional_params.pop("aws_region_name", None) or region_from_model ### SET REGION NAME ### if aws_region_name is None: @@ -188,6 +196,7 @@ class BedrockEmbedding(BaseAWSLLM): Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None + bedrock_model: Final = _without_region_path(model) # Handle async invoke responses (single response with invocationArn) if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: @@ -226,15 +235,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -408,26 +417,27 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: + region_from_model: Final = split_bedrock_region_path(model)[0] + model_path: Final = _without_region_path(model) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key) + optional_params, bearer_token=bedrock_bearer_token(api_key), region_from_model=region_from_model ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed + unencoded_model_id: Final = optional_params.pop("model_id", None) or model_path modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model, + model=model_path, model_id=unencoded_model_id, ) - # Check async invoke needs to be used - has_async_invoke: Final = "async_invoke/" in model - if has_async_invoke: - model = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(model) + has_async_invoke: Final = "async_invoke/" in model_path + bedrock_model: Final = model_path.replace("async_invoke/", "", 1) + response_model: Final = model.replace("async_invoke/", "", 1) + provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {model}. " + f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -440,9 +450,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=model, input=input, inference_params=inference_params + model=bedrock_model, input=input, inference_params=inference_params ) - elif provider == "amazon" and model in [ + elif provider == "amazon" and bedrock_model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -450,28 +460,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - model, + bedrock_model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -484,7 +494,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -526,7 +536,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=response_model, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -540,7 +550,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=response_model, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -570,7 +580,7 @@ class BedrockEmbedding(BaseAWSLLM): # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} return cohere_embedding( - model=model, + model=response_model, input=input, model_response=model_response, logging_obj=logging_obj, diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index c98844fef55..02b20050132 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -189,6 +189,77 @@ def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx. assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}] +GOVCLOUD_WEST_RUNTIME: Final = "https://bedrock-runtime.us-gov-west-1.amazonaws.com" + + +@pytest.mark.parametrize( + ("model_id", "embed_response"), + [ + ("amazon.titan-embed-text-v2:0", titan_embedding_response), + ( + "amazon.nova-2-multimodal-embeddings-v1:0", + {"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]}, + ), + ("cohere.embed-english-v3", cohere_embedding_response), + ], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_bedrock_embedding_calls_the_model_id_region_with_the_bare_model_id( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model_id: str, + embed_response: dict[str, object], + sync_mode: bool, +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/{model_id}/invoke").mock( + return_value=httpx.Response(200, json=embed_response) + ) + request: Final = { + "model": f"bedrock/us-gov-west-1/{model_id}", + "input": [test_input], + "api_key": "test-bearer-token", + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.call_count == 1 + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + + +def test_bedrock_embedding_prices_a_region_prefixed_model_from_its_region_row( + respx_mock: respx.MockRouter, +) -> None: + model: Final = "bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0" + respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + + response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token") + + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * litellm.model_cost[model]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region( + respx_mock: respx.MockRouter, +) -> None: + route = respx_mock.post( + "https://bedrock-runtime.us-gov-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) + + litellm.embedding( + model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="us-gov-east-1", + ) + + assert route.call_count == 1 + + @pytest.mark.asyncio async def test_async_bedrock_embedding_with_bearer_token(): """Test async embedding functionality with bearer token authentication""" From 9decd3f686a12e176b6204f26d6e92e3403b4250 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:19:33 -0700 Subject: [PATCH 2/6] chore(bedrock): hold the embedding fix back until the risk stamp --- litellm/llms/bedrock/embed/embedding.py | 62 +++++++++++-------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 762e9297b59..58cfb6cb95e 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -35,7 +35,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError, split_bedrock_region_path +from ..common_utils import BedrockError from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -63,18 +63,12 @@ def _sign_get_request( return request.prepare() -def _without_region_path(model: str) -> str: - region, model_id = split_bedrock_region_path(model) - return model if region is None else model_id - - class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, - region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -82,17 +76,15 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, - region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, - region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) or region_from_model + aws_region_name = optional_params.pop("aws_region_name", None) ### SET REGION NAME ### if aws_region_name is None: @@ -196,7 +188,6 @@ class BedrockEmbedding(BaseAWSLLM): Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None - bedrock_model: Final = _without_region_path(model) # Handle async invoke responses (single response with invocationArn) if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: @@ -235,15 +226,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if bedrock_model == "amazon.titan-embed-image-v1": + if model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif bedrock_model == "amazon.titan-embed-text-v1": + elif model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif bedrock_model == "amazon.titan-embed-text-v2:0": + elif model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif bedrock_model == "amazon.titan-embed-g1-text-02": + elif model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -417,27 +408,26 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: - region_from_model: Final = split_bedrock_region_path(model)[0] - model_path: Final = _without_region_path(model) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key), region_from_model=region_from_model + optional_params, bearer_token=bedrock_bearer_token(api_key) ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model_path + unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model_path, + model=model, model_id=unencoded_model_id, ) - has_async_invoke: Final = "async_invoke/" in model_path - bedrock_model: Final = model_path.replace("async_invoke/", "", 1) - response_model: Final = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(bedrock_model) + # Check async invoke needs to be used + has_async_invoke: Final = "async_invoke/" in model + if has_async_invoke: + model = model.replace("async_invoke/", "", 1) + provider: Final = self.get_bedrock_embedding_provider(model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " + f"Unable to determine bedrock embedding provider for model: {model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -450,9 +440,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=bedrock_model, input=input, inference_params=inference_params + model=model, input=input, inference_params=inference_params ) - elif provider == "amazon" and bedrock_model in [ + elif provider == "amazon" and model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -460,28 +450,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if bedrock_model == "amazon.titan-embed-image-v1": + if model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif bedrock_model == "amazon.titan-embed-text-v1": + elif model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif bedrock_model == "amazon.titan-embed-text-v2:0": + elif model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif bedrock_model == "amazon.titan-embed-g1-text-02": + elif model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - bedrock_model, + model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -494,7 +484,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -536,7 +526,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=response_model, + model=model, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -550,7 +540,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=response_model, + model=model, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -580,7 +570,7 @@ class BedrockEmbedding(BaseAWSLLM): # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} return cohere_embedding( - model=response_model, + model=model, input=input, model_response=model_response, logging_obj=logging_obj, From 79c03cd9758ce80773ba5c75c4b1eec4c848f272 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:21:01 -0700 Subject: [PATCH 3/6] fix(bedrock): send region-prefixed embedding model ids to that region without the prefix --- litellm/llms/bedrock/embed/embedding.py | 62 ++++++++++++++----------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 58cfb6cb95e..762e9297b59 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -35,7 +35,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError +from ..common_utils import BedrockError, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -63,12 +63,18 @@ def _sign_get_request( return request.prepare() +def _without_region_path(model: str) -> str: + region, model_id = split_bedrock_region_path(model) + return model if region is None else model_id + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, + region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -76,15 +82,17 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, + region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, + region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) + aws_region_name = optional_params.pop("aws_region_name", None) or region_from_model ### SET REGION NAME ### if aws_region_name is None: @@ -188,6 +196,7 @@ class BedrockEmbedding(BaseAWSLLM): Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None + bedrock_model: Final = _without_region_path(model) # Handle async invoke responses (single response with invocationArn) if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: @@ -226,15 +235,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -408,26 +417,27 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: + region_from_model: Final = split_bedrock_region_path(model)[0] + model_path: Final = _without_region_path(model) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key) + optional_params, bearer_token=bedrock_bearer_token(api_key), region_from_model=region_from_model ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed + unencoded_model_id: Final = optional_params.pop("model_id", None) or model_path modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model, + model=model_path, model_id=unencoded_model_id, ) - # Check async invoke needs to be used - has_async_invoke: Final = "async_invoke/" in model - if has_async_invoke: - model = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(model) + has_async_invoke: Final = "async_invoke/" in model_path + bedrock_model: Final = model_path.replace("async_invoke/", "", 1) + response_model: Final = model.replace("async_invoke/", "", 1) + provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {model}. " + f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -440,9 +450,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=model, input=input, inference_params=inference_params + model=bedrock_model, input=input, inference_params=inference_params ) - elif provider == "amazon" and model in [ + elif provider == "amazon" and bedrock_model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -450,28 +460,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - model, + bedrock_model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -484,7 +494,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -526,7 +536,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=response_model, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -540,7 +550,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=response_model, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -570,7 +580,7 @@ class BedrockEmbedding(BaseAWSLLM): # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} return cohere_embedding( - model=model, + model=response_model, input=input, model_response=model_response, logging_obj=logging_obj, From 3dc3e6d867ae3da602378baf658581b60a3c47ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:18:40 -0700 Subject: [PATCH 4/6] fix(bedrock): keep a model_id's region and price embeddings at the region the call goes to --- litellm/llms/bedrock/embed/embedding.py | 13 ++++++--- .../bedrock/embed/test_bedrock_embedding.py | 27 +++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 762e9297b59..3eed3d1f862 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,6 +9,7 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx +from pydantic import TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL @@ -63,6 +64,9 @@ def _sign_get_request( return request.prepare() +_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + def _without_region_path(model: str) -> str: region, model_id = split_bedrock_region_path(model) return model if region is None else model_id @@ -417,14 +421,15 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: - region_from_model: Final = split_bedrock_region_path(model)[0] + model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) + region_from_model: Final = split_bedrock_region_path(model)[0] if model_id_param is None else None model_path: Final = _without_region_path(model) credentials, aws_region_name = self._load_credentials( optional_params, bearer_token=bedrock_bearer_token(api_key), region_from_model=region_from_model ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model_path + unencoded_model_id: Final = model_id_param or model_path modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, @@ -433,7 +438,9 @@ class BedrockEmbedding(BaseAWSLLM): ) has_async_invoke: Final = "async_invoke/" in model_path bedrock_model: Final = model_path.replace("async_invoke/", "", 1) - response_model: Final = model.replace("async_invoke/", "", 1) + response_model: Final = ( + model.replace("async_invoke/", "", 1) if region_from_model is None else f"{aws_region_name}/{bedrock_model}" + ) provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 02b20050132..33541476868 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -243,14 +243,14 @@ def test_bedrock_embedding_prices_a_region_prefixed_model_from_its_region_row( ) -def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region( +def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region_for_routing_and_price( respx_mock: respx.MockRouter, ) -> None: route = respx_mock.post( "https://bedrock-runtime.us-gov-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) - litellm.embedding( + response: Final = litellm.embedding( model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", input=[test_input], api_key="test-bearer-token", @@ -258,6 +258,29 @@ def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_regio ) assert route.call_count == 1 + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens + * litellm.model_cost["bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_model_id_keeps_its_region_over_the_model_path_region( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") + route = respx_mock.post(url__startswith="https://bedrock-runtime.us-west-2.amazonaws.com/model/").mock( + return_value=httpx.Response(200, json=cohere_embedding_response) + ) + + litellm.embedding( + model="bedrock/us-east-1/cohere.embed-english-v3", + input=[test_input], + api_key="test-bearer-token", + model_id="arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-english-v3", + ) + + assert route.call_count == 1 @pytest.mark.asyncio From 4470033ccdd5114128064491923b3c10feb31ca3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:53:31 -0700 Subject: [PATCH 5/6] fix(bedrock): read a model_id ARN's region for embeddings and keep unlisted regions mappable --- litellm/llms/bedrock/embed/embedding.py | 22 ++++-- .../bedrock/embed/test_bedrock_embedding.py | 78 +++++++++++++++++-- 2 files changed, 88 insertions(+), 12 deletions(-) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 3eed3d1f862..5575de19530 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -36,7 +36,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError, split_bedrock_region_path +from ..common_utils import BedrockError, get_all_bedrock_regions, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -72,6 +72,14 @@ def _without_region_path(model: str) -> str: return model if region is None else model_id +def _response_model(model: str, path_region: str | None, routed_region: str, bedrock_model: str) -> str: + if path_region is None: + return model.replace("async_invoke/", "", 1) + if routed_region in get_all_bedrock_regions(): + return f"{routed_region}/{bedrock_model}" + return bedrock_model + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( @@ -421,11 +429,13 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: - model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) - region_from_model: Final = split_bedrock_region_path(model)[0] if model_id_param is None else None + model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None + path_region: Final = split_bedrock_region_path(model)[0] if model_id_param is None else None model_path: Final = _without_region_path(model) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key), region_from_model=region_from_model + optional_params, + bearer_token=bedrock_bearer_token(api_key), + region_from_model=path_region or self._get_aws_region_from_model_arn(model_id_param), ) ### TRANSFORMATION ### @@ -438,9 +448,7 @@ class BedrockEmbedding(BaseAWSLLM): ) has_async_invoke: Final = "async_invoke/" in model_path bedrock_model: Final = model_path.replace("async_invoke/", "", 1) - response_model: Final = ( - model.replace("async_invoke/", "", 1) if region_from_model is None else f"{aws_region_name}/{bedrock_model}" - ) + response_model: Final = _response_model(model, path_region, aws_region_name, bedrock_model) provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 33541476868..70471c3d1b5 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -264,25 +264,93 @@ def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_regio ) -def test_bedrock_embedding_model_id_keeps_its_region_over_the_model_path_region( +def test_bedrock_embedding_model_id_arn_region_wins_over_the_model_path_region_and_env( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") - route = respx_mock.post(url__startswith="https://bedrock-runtime.us-west-2.amazonaws.com/model/").mock( - return_value=httpx.Response(200, json=cohere_embedding_response) + route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock( + return_value=httpx.Response(200, json=titan_embedding_response) ) litellm.embedding( - model="bedrock/us-east-1/cohere.embed-english-v3", + model="bedrock/us-east-1/amazon.titan-embed-text-v2:0", input=[test_input], api_key="test-bearer-token", - model_id="arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-english-v3", + model_id="arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.titan-embed-text-v2:0", ) assert route.call_count == 1 +@pytest.mark.parametrize( + "model,extra_params", + [ + ("bedrock/invoke/us-gov-west-1/amazon.titan-embed-text-v2:0", {}), + ("bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", {"model_id": ""}), + ], +) +def test_bedrock_embedding_region_path_variants_call_the_path_region_with_the_bare_id_and_its_price( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + extra_params: dict[str, str], +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + + response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token", **extra_params) + + assert route.call_count == 1 + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens + * litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_explicit_region_outside_the_known_list_returns_the_vector_at_the_bare_price( + respx_mock: respx.MockRouter, +) -> None: + route = respx_mock.post( + "https://bedrock-runtime.zz-unlisted-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) + + response: Final = litellm.embedding( + model="bedrock/us-east-1/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="zz-unlisted-1", + ) + + assert route.call_count == 1 + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * litellm.model_cost["amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_that_region( + respx_mock: respx.MockRouter, +) -> None: + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/async-invoke").mock( + return_value=httpx.Response( + 200, json={"invocationArn": "arn:aws-us-gov:bedrock:us-gov-west-1:123456789012:async-invoke/abc123"} + ) + ) + + litellm.embedding( + model="bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", + input=[test_input], + api_key="test-bearer-token", + output_s3_uri="s3://test-bucket/async-invoke-output/", + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content)["modelId"] == "amazon.nova-2-multimodal-embeddings-v1:0" + + @pytest.mark.asyncio async def test_async_bedrock_embedding_with_bearer_token(): """Test async embedding functionality with bearer token authentication""" From 3a5fdc0c3cd0b425b10b7f28f89946bdd370b925 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:34:32 -0700 Subject: [PATCH 6/6] fix(bedrock): route async_invoke embeddings whether the region comes before or after it --- litellm/llms/bedrock/embed/embedding.py | 12 ++++++------ .../llms/bedrock/embed/test_bedrock_embedding.py | 10 +++++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 5575de19530..953f026e0b2 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -430,8 +430,10 @@ class BedrockEmbedding(BaseAWSLLM): api_key: str | None = None, ) -> EmbeddingResponse: model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None - path_region: Final = split_bedrock_region_path(model)[0] if model_id_param is None else None - model_path: Final = _without_region_path(model) + has_async_invoke: Final = "async_invoke/" in model + model_without_async_route: Final = model.replace("async_invoke/", "", 1) + path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None + bedrock_model: Final = _without_region_path(model_without_async_route) credentials, aws_region_name = self._load_credentials( optional_params, bearer_token=bedrock_bearer_token(api_key), @@ -439,15 +441,13 @@ class BedrockEmbedding(BaseAWSLLM): ) ### TRANSFORMATION ### - unencoded_model_id: Final = model_id_param or model_path + unencoded_model_id: Final = model_id_param or bedrock_model modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model_path, + model=bedrock_model, model_id=unencoded_model_id, ) - has_async_invoke: Final = "async_invoke/" in model_path - bedrock_model: Final = model_path.replace("async_invoke/", "", 1) response_model: Final = _response_model(model, path_region, aws_region_name, bedrock_model) provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 70471c3d1b5..d68d246d79f 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -331,8 +331,16 @@ def test_bedrock_embedding_explicit_region_outside_the_known_list_returns_the_ve ) +@pytest.mark.parametrize( + "model", + [ + "bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", + "bedrock/async_invoke/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0", + ], +) def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_that_region( respx_mock: respx.MockRouter, + model: str, ) -> None: route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/async-invoke").mock( return_value=httpx.Response( @@ -341,7 +349,7 @@ def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_ ) litellm.embedding( - model="bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", + model=model, input=[test_input], api_key="test-bearer-token", output_s3_uri="s3://test-bucket/async-invoke-output/",