mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(bedrock): keep chat's region rules for embeddings and leave ARN pricing as on main
A plain model_id skips the region in the model, as chat does, so a deployment that worked on main keeps calling the same region. An empty aws_region_name fails region validation again instead of falling back. This reverts the ARN regional pricing, since many regional rows lack the bare row's priority, batch, and long-context rates
This commit is contained in:
parent
a341c87f6f
commit
ce2813a669
3 changed files with 35 additions and 24 deletions
|
|
@ -22,7 +22,6 @@ from litellm.constants import (
|
|||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.aws_partition import is_bedrock_arn
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
|
||||
|
|
@ -885,15 +884,6 @@ def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _bedrock_invoked_profile(invoked_model_id: str) -> ModelInfo | None:
|
||||
profile: Final = _cost_map_model_info(invoked_model_id, "bedrock")
|
||||
arn_region: Final = invoked_model_id.split(":")[3] if is_bedrock_arn(invoked_model_id) else None
|
||||
regional_key: Final = f"bedrock/{arn_region}/{profile['key']}" if profile is not None and arn_region else None
|
||||
if regional_key is None or _raw_cost_map_entry(regional_key) is None:
|
||||
return profile
|
||||
return _cost_map_model_info(regional_key, "bedrock")
|
||||
|
||||
|
||||
def _bedrock_pricing_info(
|
||||
model: str | None,
|
||||
invoked_model_id: str | None,
|
||||
|
|
@ -910,7 +900,7 @@ def _bedrock_pricing_info(
|
|||
),
|
||||
None,
|
||||
)
|
||||
profile: Final = _bedrock_invoked_profile(invoked_model_id) if invoked_model_id else None
|
||||
profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None
|
||||
if profile is None or not _declares_rates(profile):
|
||||
return _cost_map_model_info(registration[0], None) if registration is not None else None
|
||||
if registration is None or registration[0] == profile["key"]:
|
||||
|
|
|
|||
|
|
@ -98,7 +98,8 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
region_from_model: str | None = None,
|
||||
) -> tuple[Credentials | None, str]:
|
||||
auth_params: Final = pop_aws_auth_params(optional_params)
|
||||
aws_region_name = optional_params.pop("aws_region_name", None) or region_from_model
|
||||
explicit_region_name: Final = optional_params.pop("aws_region_name", None)
|
||||
aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name
|
||||
|
||||
### SET REGION NAME ###
|
||||
if aws_region_name is None:
|
||||
|
|
@ -432,7 +433,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None
|
||||
has_async_invoke: Final = "async_invoke/" in model
|
||||
model_without_async_route: Final = model.replace("async_invoke/", "", 1)
|
||||
path_region: Final = split_bedrock_region_path(model_without_async_route)[0]
|
||||
path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None
|
||||
bedrock_model: Final = _without_region_path(model_without_async_route)
|
||||
credentials, aws_region_name = self._load_credentials(
|
||||
optional_params,
|
||||
|
|
|
|||
|
|
@ -283,25 +283,43 @@ def test_bedrock_embedding_model_id_arn_region_wins_over_the_model_path_region_a
|
|||
assert route.call_count == 1
|
||||
|
||||
|
||||
def test_bedrock_embedding_plain_model_id_calls_the_model_path_region_over_env(
|
||||
def test_bedrock_embedding_plain_model_id_calls_the_env_region_like_chat(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
|
||||
return_value=httpx.Response(200, json=titan_embedding_response)
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
|
||||
route = respx_mock.post("https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.embed-english-v3/invoke").mock(
|
||||
return_value=httpx.Response(200, json=cohere_embedding_response)
|
||||
)
|
||||
|
||||
litellm.embedding(
|
||||
model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0",
|
||||
model="bedrock/us-east-1/cohere.embed-english-v3",
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
model_id="amazon.titan-embed-text-v2:0",
|
||||
model_id="cohere.embed-english-v3",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
def test_bedrock_embedding_rejects_an_empty_aws_region_name_instead_of_using_the_path_region(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route = respx_mock.post(url__startswith=GOVCLOUD_WEST_RUNTIME).mock(
|
||||
return_value=httpx.Response(200, json=titan_embedding_response)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError, match="Invalid AWS region format"):
|
||||
litellm.embedding(
|
||||
model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0",
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
aws_region_name="",
|
||||
)
|
||||
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,extra_params",
|
||||
[
|
||||
|
|
@ -406,22 +424,24 @@ def test_bedrock_cohere_embedding_bills_the_row_of_the_region_it_calls(
|
|||
def test_bedrock_embedding_with_a_govcloud_arn_model_id_bills_at_the_arn_region_row(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
arn: Final = "arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.nova-2-multimodal-embeddings-v1:0"
|
||||
route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock(
|
||||
return_value=httpx.Response(200, json=titan_embedding_response)
|
||||
return_value=httpx.Response(
|
||||
200, json={"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]}
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.embedding(
|
||||
model="bedrock/amazon.titan-embed-text-v2:0",
|
||||
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
input=[test_input],
|
||||
api_key="test-bearer-token",
|
||||
model_id="arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.titan-embed-text-v2:0",
|
||||
model_id=arn,
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens
|
||||
* litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
|
||||
* litellm.model_cost["bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0"]["input_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue