mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge 3a5fdc0c3c into e04e89c701
This commit is contained in:
commit
30d00af9c9
2 changed files with 221 additions and 26 deletions
|
|
@ -9,6 +9,7 @@ from collections.abc import Callable, Mapping
|
|||
from typing import TYPE_CHECKING, Final, get_args, overload
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
|
||||
|
|
@ -36,7 +37,7 @@ from ..base_aws_llm import (
|
|||
pop_aws_auth_params,
|
||||
run_aws_signing,
|
||||
)
|
||||
from ..common_utils import BedrockError
|
||||
from ..common_utils import BedrockError, get_all_bedrock_regions, split_bedrock_region_path
|
||||
from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
|
||||
from .amazon_titan_g1_transformation import AmazonTitanG1Config
|
||||
from .amazon_titan_multimodal_transformation import (
|
||||
|
|
@ -64,12 +65,29 @@ def _sign_get_request(
|
|||
return request.prepare()
|
||||
|
||||
|
||||
_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||
|
||||
|
||||
def _without_region_path(model: str) -> str:
|
||||
region, model_id = split_bedrock_region_path(model)
|
||||
return model if region is None else model_id
|
||||
|
||||
|
||||
def _response_model(model: str, path_region: str | None, routed_region: str, bedrock_model: str) -> str:
|
||||
if path_region is None:
|
||||
return model.replace("async_invoke/", "", 1)
|
||||
if routed_region in get_all_bedrock_regions():
|
||||
return f"{routed_region}/{bedrock_model}"
|
||||
return bedrock_model
|
||||
|
||||
|
||||
class BedrockEmbedding(BaseAWSLLM):
|
||||
@overload
|
||||
def _load_credentials(
|
||||
self,
|
||||
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
|
||||
bearer_token: None = None,
|
||||
region_from_model: str | None = None,
|
||||
) -> tuple[Credentials, str]: ...
|
||||
|
||||
@overload
|
||||
|
|
@ -77,15 +95,17 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
self,
|
||||
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
|
||||
bearer_token: str,
|
||||
region_from_model: str | None = None,
|
||||
) -> tuple[None, str]: ...
|
||||
|
||||
def _load_credentials(
|
||||
self,
|
||||
optional_params: dict,
|
||||
bearer_token: str | None = None,
|
||||
region_from_model: str | None = None,
|
||||
) -> tuple[Credentials | None, str]:
|
||||
auth_params: Final = pop_aws_auth_params(optional_params)
|
||||
aws_region_name = optional_params.pop("aws_region_name", None)
|
||||
aws_region_name = optional_params.pop("aws_region_name", None) or region_from_model
|
||||
|
||||
### SET REGION NAME ###
|
||||
if aws_region_name is None:
|
||||
|
|
@ -189,6 +209,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
Transforms the response from the Bedrock embedding provider to the OpenAI format.
|
||||
"""
|
||||
returned_response: EmbeddingResponse | None = None
|
||||
bedrock_model: Final = _without_region_path(model)
|
||||
|
||||
# Handle async invoke responses (single response with invocationArn)
|
||||
if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]:
|
||||
|
|
@ -227,15 +248,15 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
)
|
||||
else:
|
||||
# Handle regular invoke responses
|
||||
if model == "amazon.titan-embed-image-v1":
|
||||
if bedrock_model == "amazon.titan-embed-image-v1":
|
||||
returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response(
|
||||
response_list=response_list, model=model, batch_data=batch_data
|
||||
)
|
||||
elif model == "amazon.titan-embed-text-v1":
|
||||
elif bedrock_model == "amazon.titan-embed-text-v1":
|
||||
returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
|
||||
elif model == "amazon.titan-embed-text-v2:0":
|
||||
elif bedrock_model == "amazon.titan-embed-text-v2:0":
|
||||
returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model)
|
||||
elif model == "amazon.titan-embed-g1-text-02":
|
||||
elif bedrock_model == "amazon.titan-embed-g1-text-02":
|
||||
returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
|
||||
elif provider == "twelvelabs":
|
||||
returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response(
|
||||
|
|
@ -409,26 +430,30 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
) -> EmbeddingResponse:
|
||||
model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None
|
||||
has_async_invoke: Final = "async_invoke/" in model
|
||||
model_without_async_route: Final = model.replace("async_invoke/", "", 1)
|
||||
path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None
|
||||
bedrock_model: Final = _without_region_path(model_without_async_route)
|
||||
credentials, aws_region_name = self._load_credentials(
|
||||
optional_params, bearer_token=bedrock_bearer_token(api_key)
|
||||
optional_params,
|
||||
bearer_token=bedrock_bearer_token(api_key),
|
||||
region_from_model=path_region or self._get_aws_region_from_model_arn(model_id_param),
|
||||
)
|
||||
|
||||
### TRANSFORMATION ###
|
||||
unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed
|
||||
unencoded_model_id: Final = model_id_param or bedrock_model
|
||||
modelId: Final = urllib.parse.quote(unencoded_model_id, safe="")
|
||||
aws_region_name = self._get_aws_region_name(
|
||||
optional_params={"aws_region_name": aws_region_name},
|
||||
model=model,
|
||||
model=bedrock_model,
|
||||
model_id=unencoded_model_id,
|
||||
)
|
||||
# Check async invoke needs to be used
|
||||
has_async_invoke: Final = "async_invoke/" in model
|
||||
if has_async_invoke:
|
||||
model = model.replace("async_invoke/", "", 1)
|
||||
provider: Final = self.get_bedrock_embedding_provider(model)
|
||||
response_model: Final = _response_model(model, path_region, aws_region_name, bedrock_model)
|
||||
provider: Final = self.get_bedrock_embedding_provider(bedrock_model)
|
||||
if provider is None:
|
||||
raise Exception(
|
||||
f"Unable to determine bedrock embedding provider for model: {model}. "
|
||||
f"Unable to determine bedrock embedding provider for model: {bedrock_model}. "
|
||||
f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
|
||||
)
|
||||
inference_params = copy.deepcopy(optional_params)
|
||||
|
|
@ -441,9 +466,9 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
batch_data: list | None = None
|
||||
if provider == "cohere":
|
||||
data = BedrockCohereEmbeddingConfig().transform_request(
|
||||
model=model, input=input, inference_params=inference_params
|
||||
model=bedrock_model, input=input, inference_params=inference_params
|
||||
)
|
||||
elif provider == "amazon" and model in [
|
||||
elif provider == "amazon" and bedrock_model in [
|
||||
"amazon.titan-embed-image-v1",
|
||||
"amazon.titan-embed-text-v1",
|
||||
"amazon.titan-embed-text-v2:0",
|
||||
|
|
@ -451,28 +476,28 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
]:
|
||||
batch_data = []
|
||||
for i in input:
|
||||
if model == "amazon.titan-embed-image-v1":
|
||||
if bedrock_model == "amazon.titan-embed-image-v1":
|
||||
transformed_request: AmazonEmbeddingRequest = (
|
||||
AmazonTitanMultimodalEmbeddingG1Config().transform_request(
|
||||
input=i, inference_params=inference_params
|
||||
)
|
||||
)
|
||||
elif model == "amazon.titan-embed-text-v1":
|
||||
elif bedrock_model == "amazon.titan-embed-text-v1":
|
||||
transformed_request = AmazonTitanG1Config().transform_request(
|
||||
input=i, inference_params=inference_params
|
||||
)
|
||||
elif model == "amazon.titan-embed-text-v2:0":
|
||||
elif bedrock_model == "amazon.titan-embed-text-v2:0":
|
||||
transformed_request = AmazonTitanV2Config().transform_request(
|
||||
input=i, inference_params=inference_params
|
||||
)
|
||||
elif model == "amazon.titan-embed-g1-text-02":
|
||||
elif bedrock_model == "amazon.titan-embed-g1-text-02":
|
||||
transformed_request = AmazonTitanG1Config().transform_request(
|
||||
input=i, inference_params=inference_params
|
||||
)
|
||||
else:
|
||||
raise Exception(
|
||||
"Unmapped model. Received={}. Expected={}".format(
|
||||
model,
|
||||
bedrock_model,
|
||||
[
|
||||
"amazon.titan-embed-image-v1",
|
||||
"amazon.titan-embed-text-v1",
|
||||
|
|
@ -485,7 +510,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
elif provider == "twelvelabs":
|
||||
batch_data = []
|
||||
for i in input:
|
||||
twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request(
|
||||
twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request(
|
||||
input=i,
|
||||
inference_params=inference_params,
|
||||
async_invoke_route=has_async_invoke,
|
||||
|
|
@ -527,7 +552,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
extra_headers=extra_headers,
|
||||
endpoint_url=endpoint_url,
|
||||
aws_region_name=aws_region_name,
|
||||
model=model,
|
||||
model=response_model,
|
||||
logging_obj=logging_obj,
|
||||
api_key=api_key,
|
||||
provider=provider,
|
||||
|
|
@ -541,7 +566,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
extra_headers=extra_headers,
|
||||
endpoint_url=endpoint_url,
|
||||
aws_region_name=aws_region_name,
|
||||
model=model,
|
||||
model=response_model,
|
||||
logging_obj=logging_obj,
|
||||
api_key=api_key,
|
||||
provider=provider,
|
||||
|
|
@ -571,7 +596,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
# Convert CaseInsensitiveDict to regular dict for httpx compatibility
|
||||
headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {}
|
||||
return cohere_embedding(
|
||||
model=model,
|
||||
model=response_model,
|
||||
input=input,
|
||||
model_response=model_response,
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -189,6 +189,176 @@ def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx.
|
|||
assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}]
|
||||
|
||||
|
||||
GOVCLOUD_WEST_RUNTIME: Final = "https://bedrock-runtime.us-gov-west-1.amazonaws.com"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "embed_response"),
|
||||
[
|
||||
("amazon.titan-embed-text-v2:0", titan_embedding_response),
|
||||
(
|
||||
"amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
{"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]},
|
||||
),
|
||||
("cohere.embed-english-v3", cohere_embedding_response),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
async def test_bedrock_embedding_calls_the_model_id_region_with_the_bare_model_id(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_id: str,
|
||||
embed_response: dict[str, object],
|
||||
sync_mode: bool,
|
||||
) -> None:
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/{model_id}/invoke").mock(
|
||||
return_value=httpx.Response(200, json=embed_response)
|
||||
)
|
||||
request: Final = {
|
||||
"model": f"bedrock/us-gov-west-1/{model_id}",
|
||||
"input": [test_input],
|
||||
"api_key": "test-bearer-token",
|
||||
}
|
||||
|
||||
response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_bedrock_embedding_prices_a_region_prefixed_model_from_its_region_row(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
model: Final = "bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"
|
||||
respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
|
||||
return_value=httpx.Response(200, json=titan_embedding_response)
|
||||
)
|
||||
|
||||
response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token")
|
||||
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens * litellm.model_cost[model]["input_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region_for_routing_and_price(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route = respx_mock.post(
|
||||
"https://bedrock-runtime.us-gov-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke"
|
||||
).mock(return_value=httpx.Response(200, json=titan_embedding_response))
|
||||
|
||||
response: Final = litellm.embedding(
|
||||
model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0",
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
aws_region_name="us-gov-east-1",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens
|
||||
* litellm.model_cost["bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_bedrock_embedding_model_id_arn_region_wins_over_the_model_path_region_and_env(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
|
||||
route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock(
|
||||
return_value=httpx.Response(200, json=titan_embedding_response)
|
||||
)
|
||||
|
||||
litellm.embedding(
|
||||
model="bedrock/us-east-1/amazon.titan-embed-text-v2:0",
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
model_id="arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.titan-embed-text-v2:0",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,extra_params",
|
||||
[
|
||||
("bedrock/invoke/us-gov-west-1/amazon.titan-embed-text-v2:0", {}),
|
||||
("bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", {"model_id": ""}),
|
||||
],
|
||||
)
|
||||
def test_bedrock_embedding_region_path_variants_call_the_path_region_with_the_bare_id_and_its_price(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
extra_params: dict[str, str],
|
||||
) -> None:
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
|
||||
return_value=httpx.Response(200, json=titan_embedding_response)
|
||||
)
|
||||
|
||||
response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token", **extra_params)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens
|
||||
* litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_bedrock_embedding_explicit_region_outside_the_known_list_returns_the_vector_at_the_bare_price(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route = respx_mock.post(
|
||||
"https://bedrock-runtime.zz-unlisted-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke"
|
||||
).mock(return_value=httpx.Response(200, json=titan_embedding_response))
|
||||
|
||||
response: Final = litellm.embedding(
|
||||
model="bedrock/us-east-1/amazon.titan-embed-text-v2:0",
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
aws_region_name="zz-unlisted-1",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens * litellm.model_cost["amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
"bedrock/async_invoke/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
],
|
||||
)
|
||||
def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_that_region(
|
||||
respx_mock: respx.MockRouter,
|
||||
model: str,
|
||||
) -> None:
|
||||
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/async-invoke").mock(
|
||||
return_value=httpx.Response(
|
||||
200, json={"invocationArn": "arn:aws-us-gov:bedrock:us-gov-west-1:123456789012:async-invoke/abc123"}
|
||||
)
|
||||
)
|
||||
|
||||
litellm.embedding(
|
||||
model=model,
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
output_s3_uri="s3://test-bucket/async-invoke-output/",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content)["modelId"] == "amazon.nova-2-multimodal-embeddings-v1:0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_bedrock_embedding_with_bearer_token():
|
||||
"""Test async embedding functionality with bearer token authentication"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue