This commit is contained in:
Mateo Wang 2026-10-24 06:16:11 +00:00 • committed by GitHub
commit 30d00af9c9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 221 additions and 26 deletions

View file

@ -9,6 +9,7 @@ from collections.abc import Callable, Mapping
from typing import TYPE_CHECKING, Final, get_args, overload
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
@ -36,7 +37,7 @@ from ..base_aws_llm import (
pop_aws_auth_params,
run_aws_signing,
)
from ..common_utils import BedrockError
from ..common_utils import BedrockError, get_all_bedrock_regions, split_bedrock_region_path
from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
from .amazon_titan_g1_transformation import AmazonTitanG1Config
from .amazon_titan_multimodal_transformation import (
@ -64,12 +65,29 @@ def _sign_get_request(
return request.prepare()
_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
def _without_region_path(model: str) -> str:
region, model_id = split_bedrock_region_path(model)
return model if region is None else model_id
def _response_model(model: str, path_region: str | None, routed_region: str, bedrock_model: str) -> str:
if path_region is None:
return model.replace("async_invoke/", "", 1)
if routed_region in get_all_bedrock_regions():
return f"{routed_region}/{bedrock_model}"
return bedrock_model
class BedrockEmbedding(BaseAWSLLM):
@overload
def _load_credentials(
self,
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
bearer_token: None = None,
region_from_model: str | None = None,
) -> tuple[Credentials, str]: ...
@overload
@ -77,15 +95,17 @@ class BedrockEmbedding(BaseAWSLLM):
self,
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
bearer_token: str,
region_from_model: str | None = None,
) -> tuple[None, str]: ...
def _load_credentials(
self,
optional_params: dict,
bearer_token: str | None = None,
region_from_model: str | None = None,
) -> tuple[Credentials | None, str]:
auth_params: Final = pop_aws_auth_params(optional_params)
aws_region_name = optional_params.pop("aws_region_name", None)
aws_region_name = optional_params.pop("aws_region_name", None) or region_from_model
### SET REGION NAME ###
if aws_region_name is None:
@ -189,6 +209,7 @@ class BedrockEmbedding(BaseAWSLLM):
Transforms the response from the Bedrock embedding provider to the OpenAI format.
"""
returned_response: EmbeddingResponse | None = None
bedrock_model: Final = _without_region_path(model)
# Handle async invoke responses (single response with invocationArn)
if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]:
@ -227,15 +248,15 @@ class BedrockEmbedding(BaseAWSLLM):
)
else:
# Handle regular invoke responses
if model == "amazon.titan-embed-image-v1":
if bedrock_model == "amazon.titan-embed-image-v1":
returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response(
response_list=response_list, model=model, batch_data=batch_data
)
elif model == "amazon.titan-embed-text-v1":
elif bedrock_model == "amazon.titan-embed-text-v1":
returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
elif model == "amazon.titan-embed-text-v2:0":
elif bedrock_model == "amazon.titan-embed-text-v2:0":
returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model)
elif model == "amazon.titan-embed-g1-text-02":
elif bedrock_model == "amazon.titan-embed-g1-text-02":
returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
elif provider == "twelvelabs":
returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response(
@ -409,26 +430,30 @@ class BedrockEmbedding(BaseAWSLLM):
litellm_params: dict,
api_key: str | None = None,
) -> EmbeddingResponse:
model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None
has_async_invoke: Final = "async_invoke/" in model
model_without_async_route: Final = model.replace("async_invoke/", "", 1)
path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None
bedrock_model: Final = _without_region_path(model_without_async_route)
credentials, aws_region_name = self._load_credentials(
optional_params, bearer_token=bedrock_bearer_token(api_key)
optional_params,
bearer_token=bedrock_bearer_token(api_key),
region_from_model=path_region or self._get_aws_region_from_model_arn(model_id_param),
)
### TRANSFORMATION ###
unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed
unencoded_model_id: Final = model_id_param or bedrock_model
modelId: Final = urllib.parse.quote(unencoded_model_id, safe="")
aws_region_name = self._get_aws_region_name(
optional_params={"aws_region_name": aws_region_name},
model=model,
model=bedrock_model,
model_id=unencoded_model_id,
)
# Check async invoke needs to be used
has_async_invoke: Final = "async_invoke/" in model
if has_async_invoke:
model = model.replace("async_invoke/", "", 1)
provider: Final = self.get_bedrock_embedding_provider(model)
response_model: Final = _response_model(model, path_region, aws_region_name, bedrock_model)
provider: Final = self.get_bedrock_embedding_provider(bedrock_model)
if provider is None:
raise Exception(
f"Unable to determine bedrock embedding provider for model: {model}. "
f"Unable to determine bedrock embedding provider for model: {bedrock_model}. "
f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
)
inference_params = copy.deepcopy(optional_params)
@ -441,9 +466,9 @@ class BedrockEmbedding(BaseAWSLLM):
batch_data: list | None = None
if provider == "cohere":
data = BedrockCohereEmbeddingConfig().transform_request(
model=model, input=input, inference_params=inference_params
model=bedrock_model, input=input, inference_params=inference_params
)
elif provider == "amazon" and model in [
elif provider == "amazon" and bedrock_model in [
"amazon.titan-embed-image-v1",
"amazon.titan-embed-text-v1",
"amazon.titan-embed-text-v2:0",
@ -451,28 +476,28 @@ class BedrockEmbedding(BaseAWSLLM):
]:
batch_data = []
for i in input:
if model == "amazon.titan-embed-image-v1":
if bedrock_model == "amazon.titan-embed-image-v1":
transformed_request: AmazonEmbeddingRequest = (
AmazonTitanMultimodalEmbeddingG1Config().transform_request(
input=i, inference_params=inference_params
)
)
elif model == "amazon.titan-embed-text-v1":
elif bedrock_model == "amazon.titan-embed-text-v1":
transformed_request = AmazonTitanG1Config().transform_request(
input=i, inference_params=inference_params
)
elif model == "amazon.titan-embed-text-v2:0":
elif bedrock_model == "amazon.titan-embed-text-v2:0":
transformed_request = AmazonTitanV2Config().transform_request(
input=i, inference_params=inference_params
)
elif model == "amazon.titan-embed-g1-text-02":
elif bedrock_model == "amazon.titan-embed-g1-text-02":
transformed_request = AmazonTitanG1Config().transform_request(
input=i, inference_params=inference_params
)
else:
raise Exception(
"Unmapped model. Received={}. Expected={}".format(
model,
bedrock_model,
[
"amazon.titan-embed-image-v1",
"amazon.titan-embed-text-v1",
@ -485,7 +510,7 @@ class BedrockEmbedding(BaseAWSLLM):
elif provider == "twelvelabs":
batch_data = []
for i in input:
twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request(
twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request(
input=i,
inference_params=inference_params,
async_invoke_route=has_async_invoke,
@ -527,7 +552,7 @@ class BedrockEmbedding(BaseAWSLLM):
extra_headers=extra_headers,
endpoint_url=endpoint_url,
aws_region_name=aws_region_name,
model=model,
model=response_model,
logging_obj=logging_obj,
api_key=api_key,
provider=provider,
@ -541,7 +566,7 @@ class BedrockEmbedding(BaseAWSLLM):
extra_headers=extra_headers,
endpoint_url=endpoint_url,
aws_region_name=aws_region_name,
model=model,
model=response_model,
logging_obj=logging_obj,
api_key=api_key,
provider=provider,
@ -571,7 +596,7 @@ class BedrockEmbedding(BaseAWSLLM):
# Convert CaseInsensitiveDict to regular dict for httpx compatibility
headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {}
return cohere_embedding(
model=model,
model=response_model,
input=input,
model_response=model_response,
logging_obj=logging_obj,

View file

@ -189,6 +189,176 @@ def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx.
assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}]
GOVCLOUD_WEST_RUNTIME: Final = "https://bedrock-runtime.us-gov-west-1.amazonaws.com"
@pytest.mark.parametrize(
("model_id", "embed_response"),
[
("amazon.titan-embed-text-v2:0", titan_embedding_response),
(
"amazon.nova-2-multimodal-embeddings-v1:0",
{"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]},
),
("cohere.embed-english-v3", cohere_embedding_response),
],
)
@pytest.mark.parametrize("sync_mode", [True, False])
async def test_bedrock_embedding_calls_the_model_id_region_with_the_bare_model_id(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
model_id: str,
embed_response: dict[str, object],
sync_mode: bool,
) -> None:
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/{model_id}/invoke").mock(
return_value=httpx.Response(200, json=embed_response)
)
request: Final = {
"model": f"bedrock/us-gov-west-1/{model_id}",
"input": [test_input],
"api_key": "test-bearer-token",
}
response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request)
assert route.call_count == 1
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
def test_bedrock_embedding_prices_a_region_prefixed_model_from_its_region_row(
respx_mock: respx.MockRouter,
) -> None:
model: Final = "bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"
respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
return_value=httpx.Response(200, json=titan_embedding_response)
)
response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token")
assert response._hidden_params["response_cost"] == pytest.approx(
response.usage.prompt_tokens * litellm.model_cost[model]["input_cost_per_token"]
)
def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region_for_routing_and_price(
respx_mock: respx.MockRouter,
) -> None:
route = respx_mock.post(
"https://bedrock-runtime.us-gov-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke"
).mock(return_value=httpx.Response(200, json=titan_embedding_response))
response: Final = litellm.embedding(
model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0",
input=[test_input],
api_key="test-bearer-token",
aws_region_name="us-gov-east-1",
)
assert route.call_count == 1
assert response._hidden_params["response_cost"] == pytest.approx(
response.usage.prompt_tokens
* litellm.model_cost["bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
)
def test_bedrock_embedding_model_id_arn_region_wins_over_the_model_path_region_and_env(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock(
return_value=httpx.Response(200, json=titan_embedding_response)
)
litellm.embedding(
model="bedrock/us-east-1/amazon.titan-embed-text-v2:0",
input=[test_input],
api_key="test-bearer-token",
model_id="arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.titan-embed-text-v2:0",
)
assert route.call_count == 1
@pytest.mark.parametrize(
"model,extra_params",
[
("bedrock/invoke/us-gov-west-1/amazon.titan-embed-text-v2:0", {}),
("bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", {"model_id": ""}),
],
)
def test_bedrock_embedding_region_path_variants_call_the_path_region_with_the_bare_id_and_its_price(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
model: str,
extra_params: dict[str, str],
) -> None:
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
return_value=httpx.Response(200, json=titan_embedding_response)
)
response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token", **extra_params)
assert route.call_count == 1
assert response._hidden_params["response_cost"] == pytest.approx(
response.usage.prompt_tokens
* litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
)
def test_bedrock_embedding_explicit_region_outside_the_known_list_returns_the_vector_at_the_bare_price(
respx_mock: respx.MockRouter,
) -> None:
route = respx_mock.post(
"https://bedrock-runtime.zz-unlisted-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke"
).mock(return_value=httpx.Response(200, json=titan_embedding_response))
response: Final = litellm.embedding(
model="bedrock/us-east-1/amazon.titan-embed-text-v2:0",
input=[test_input],
api_key="test-bearer-token",
aws_region_name="zz-unlisted-1",
)
assert route.call_count == 1
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
assert response._hidden_params["response_cost"] == pytest.approx(
response.usage.prompt_tokens * litellm.model_cost["amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
)
@pytest.mark.parametrize(
"model",
[
"bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0",
"bedrock/async_invoke/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0",
],
)
def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_that_region(
respx_mock: respx.MockRouter,
model: str,
) -> None:
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/async-invoke").mock(
return_value=httpx.Response(
200, json={"invocationArn": "arn:aws-us-gov:bedrock:us-gov-west-1:123456789012:async-invoke/abc123"}
)
)
litellm.embedding(
model=model,
input=[test_input],
api_key="test-bearer-token",
output_s3_uri="s3://test-bucket/async-invoke-output/",
)
assert route.call_count == 1
assert json.loads(route.calls[0].request.content)["modelId"] == "amazon.nova-2-multimodal-embeddings-v1:0"
@pytest.mark.asyncio
async def test_async_bedrock_embedding_with_bearer_token():
"""Test async embedding functionality with bearer token authentication"""