mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
Merge pull request #39846 from BerriAI/litellm_bedrock_mantle_govcloud_cost_row
Some checks are pending
ai-gateway image / ai-gateway release image (push) Waiting to run
CI Coverage / assert-ci-coverage (push) Waiting to run
CodSpeed Benchmarks / benchmarks (push) Waiting to run
Publish basedpyright base counts / publish (push) Waiting to run
Code Quality Checks / code-quality (push) Waiting to run
Code Quality Checks / python-310-import-smoke (push) Waiting to run
Unit Tests: Proxy DB Operations / auth-checks (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / budgets (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / custom-logging (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / db-and-spend (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / endpoints-and-responses (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / guardrails-hooks (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / jwt-and-keys (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / key-generation (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / logging-misc (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / proxy-runtime (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / proxy-server-core (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / proxy-utils (push) Blocked by required conditions
Unit Tests / proxy-endpoints (push) Waiting to run
Unit Tests / proxy-extras (push) Waiting to run
Unit Tests / caching-local (push) Waiting to run
Unit Tests / core-utils (push) Waiting to run
Unit Tests / enterprise-package (push) Waiting to run
Unit Tests / enterprise-routing (push) Waiting to run
Unit Tests / integrations (push) Waiting to run
Unit Tests / All Other Providers (push) Waiting to run
Unit Tests / Vertex AI (push) Waiting to run
Unit Tests / proxy-infra (push) Waiting to run
Unit Tests / proxy-server (push) Waiting to run
GitHub Actions Security Analysis / zizmor (push) Waiting to run
UI Unit Tests / ui-unit-tests (push) Waiting to run
Postgres Tests / proxy-security (push) Waiting to run
Postgres Tests / schema-migration (push) Waiting to run
Postgres Tests / proxy-behavior (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
Unit Tests: Documentation Validation / documentation (push) Waiting to run
Unit Tests: Proxy DB Operations / assert-shard-coverage (push) Waiting to run
Unit Tests / misc (push) Waiting to run
Unit Tests / proxy-auth (push) Waiting to run
Unit Tests / responses-caching-types (push) Waiting to run
Some checks are pending
ai-gateway image / ai-gateway release image (push) Waiting to run
CI Coverage / assert-ci-coverage (push) Waiting to run
CodSpeed Benchmarks / benchmarks (push) Waiting to run
Publish basedpyright base counts / publish (push) Waiting to run
Code Quality Checks / code-quality (push) Waiting to run
Code Quality Checks / python-310-import-smoke (push) Waiting to run
Unit Tests: Proxy DB Operations / auth-checks (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / budgets (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / custom-logging (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / db-and-spend (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / endpoints-and-responses (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / guardrails-hooks (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / jwt-and-keys (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / key-generation (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / logging-misc (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / proxy-runtime (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / proxy-server-core (push) Blocked by required conditions
Unit Tests: Proxy DB Operations / proxy-utils (push) Blocked by required conditions
Unit Tests / proxy-endpoints (push) Waiting to run
Unit Tests / proxy-extras (push) Waiting to run
Unit Tests / caching-local (push) Waiting to run
Unit Tests / core-utils (push) Waiting to run
Unit Tests / enterprise-package (push) Waiting to run
Unit Tests / enterprise-routing (push) Waiting to run
Unit Tests / integrations (push) Waiting to run
Unit Tests / All Other Providers (push) Waiting to run
Unit Tests / Vertex AI (push) Waiting to run
Unit Tests / proxy-infra (push) Waiting to run
Unit Tests / proxy-server (push) Waiting to run
GitHub Actions Security Analysis / zizmor (push) Waiting to run
UI Unit Tests / ui-unit-tests (push) Waiting to run
Postgres Tests / proxy-security (push) Waiting to run
Postgres Tests / schema-migration (push) Waiting to run
Postgres Tests / proxy-behavior (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
Unit Tests: Documentation Validation / documentation (push) Waiting to run
Unit Tests: Proxy DB Operations / assert-shard-coverage (push) Waiting to run
Unit Tests / misc (push) Waiting to run
Unit Tests / proxy-auth (push) Waiting to run
Unit Tests / responses-caching-types (push) Waiting to run
fix(bedrock_mantle): price GovCloud regions from the regional cost row and accept region-prefixed model names
This commit is contained in:
commit
c2c2a623c0
8 changed files with 334 additions and 11 deletions
|
|
@ -481,7 +481,8 @@ def cost_per_token(
|
|||
else:
|
||||
model_with_provider = f"{custom_llm_provider}/{model}"
|
||||
if region_name is not None:
|
||||
model_with_provider_and_region: Final = f"{custom_llm_provider}/{region_name}/{model}"
|
||||
bare_model: Final = model[len(_prov_prefix) :] if model_is_str and model.startswith(_prov_prefix) else model
|
||||
model_with_provider_and_region: Final = f"{custom_llm_provider}/{region_name}/{bare_model}"
|
||||
if model_with_provider_and_region in model_cost_ref: # use region based pricing, if it's available
|
||||
model_with_provider = model_with_provider_and_region
|
||||
else:
|
||||
|
|
@ -778,6 +779,7 @@ def _select_model_name_for_cost_calc(
|
|||
custom_pricing: bool | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
router_model_id: str | None = None,
|
||||
region_name: str | None = None,
|
||||
) -> str | None:
|
||||
"""
|
||||
1. If custom pricing is true, return received model name
|
||||
|
|
@ -799,8 +801,8 @@ def _select_model_name_for_cost_calc(
|
|||
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
|
||||
region_name: Final = (
|
||||
_get_hidden_str_for_cost_calc(hidden_params, "region_name")
|
||||
priced_region: Final = (
|
||||
_get_hidden_str_for_cost_calc(hidden_params, "region_name") or region_name
|
||||
if not explicit_pricing and priced_from_response
|
||||
else None
|
||||
)
|
||||
|
|
@ -837,8 +839,10 @@ def _select_model_name_for_cost_calc(
|
|||
and custom_llm_provider is not None
|
||||
and not _model_contains_known_llm_provider(return_model)
|
||||
): # add provider prefix if not already present, to match model_cost
|
||||
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
|
||||
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name)
|
||||
provider_prefix: Final = (
|
||||
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
|
||||
)
|
||||
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
|
||||
|
||||
return return_model
|
||||
|
||||
|
|
@ -1300,6 +1304,7 @@ def completion_cost(
|
|||
|
||||
service_tier = _normalize_service_tier(service_tier)
|
||||
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
selected_model: Final = _select_model_name_for_cost_calc(
|
||||
model=model,
|
||||
completion_response=completion_response,
|
||||
|
|
@ -1307,6 +1312,7 @@ def completion_cost(
|
|||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
router_model_id=router_model_id,
|
||||
region_name=region_name,
|
||||
)
|
||||
|
||||
potential_model_names: Final = [
|
||||
|
|
@ -1651,7 +1657,7 @@ def completion_cost(
|
|||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
response_time_ms=total_time,
|
||||
region_name=region_name,
|
||||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
prompt_characters=prompt_characters,
|
||||
|
|
@ -1861,6 +1867,7 @@ def response_cost_calculator(
|
|||
data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us")
|
||||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -1894,6 +1901,7 @@ def response_cost_calculator(
|
|||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
)
|
||||
return response_cost
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -601,12 +601,15 @@ def _get_openai_compatible_provider_info(
|
|||
dynamic_api_key,
|
||||
) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "bedrock_mantle":
|
||||
from litellm.llms.bedrock_mantle.common_utils import split_mantle_region_prefix
|
||||
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key, litellm_params=litellm_params, model=model
|
||||
)
|
||||
model = split_mantle_region_prefix(model)[1] # rebind-ok: the prefix is routing only, not a Mantle model id
|
||||
elif custom_llm_provider == "nvidia_nim":
|
||||
# nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
|
||||
api_base = api_base or get_secret("NVIDIA_NIM_API_BASE") or "https://integrate.api.nvidia.com/v1"
|
||||
|
|
|
|||
|
|
@ -454,6 +454,17 @@ def _resolve_vertex_location_for_cost(
|
|||
return VertexBase.get_vertex_region(configured_location, model)
|
||||
|
||||
|
||||
def _resolve_mantle_region_for_cost(
|
||||
custom_llm_provider: str | None,
|
||||
litellm_params: Mapping[str, object] | None,
|
||||
) -> str | None:
|
||||
if custom_llm_provider != "bedrock_mantle":
|
||||
return None
|
||||
from litellm.llms.bedrock_mantle.common_utils import resolve_mantle_region
|
||||
|
||||
return resolve_mantle_region(litellm_params or MappingProxyType({}))
|
||||
|
||||
|
||||
def _provider_response_id(source: object) -> str | None:
|
||||
candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
|
||||
return candidate if isinstance(candidate, str) and candidate else None
|
||||
|
|
@ -1768,6 +1779,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
optional_params=self.optional_params,
|
||||
model=litellm_model_name or self.model,
|
||||
),
|
||||
"region_name": _resolve_mantle_region_for_cost(
|
||||
custom_llm_provider=self.model_call_details.get("custom_llm_provider", None),
|
||||
litellm_params=self.model_call_details.get("litellm_params"),
|
||||
),
|
||||
}
|
||||
except Exception as e: # error creating kwargs for cost calculation
|
||||
debug_info = StandardLoggingModelCostFailureDebugInformation(
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from ...base_llm.chat.transformation import BaseLLMException
|
||||
from ...bedrock.common_utils import BedrockError
|
||||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from ..common_utils import mantle_base_segment
|
||||
from ..common_utils import mantle_base_segment, split_mantle_region_prefix
|
||||
|
||||
|
||||
class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
|
||||
|
|
@ -61,8 +61,10 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
|
|||
litellm_params: GenericLiteLLMParams | None = None,
|
||||
model: str | None = None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
prefix_region, base_model = split_mantle_region_prefix(model) if model else (None, None)
|
||||
region: Final = (
|
||||
(litellm_params.aws_region_name if litellm_params else None)
|
||||
or prefix_region
|
||||
or get_secret_str("BEDROCK_MANTLE_REGION")
|
||||
or get_secret_str("AWS_REGION_NAME")
|
||||
or get_secret_str("AWS_REGION")
|
||||
|
|
@ -75,7 +77,7 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
|
|||
api_base = (
|
||||
api_base
|
||||
or get_secret_str("BEDROCK_MANTLE_API_BASE")
|
||||
or f"https://bedrock-mantle.{region}.api.aws/{mantle_base_segment(model, litellm.model_cost)}"
|
||||
or f"https://bedrock-mantle.{region}.api.aws/{mantle_base_segment(base_model, litellm.model_cost)}"
|
||||
)
|
||||
dynamic_api_key: Final = self._resolve_bearer_token(api_key)
|
||||
return api_base, dynamic_api_key
|
||||
|
|
|
|||
|
|
@ -24,9 +24,11 @@ from botocore.exceptions import (
|
|||
)
|
||||
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS
|
||||
from litellm.llms.bedrock.common_utils import AmazonBedrockGlobalConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1"
|
||||
BEDROCK_REGIONS: Final = frozenset(AmazonBedrockGlobalConfig().get_all_regions())
|
||||
|
||||
# Standard Mantle host: https://bedrock-mantle.<region>.api.aws (group 1 = region).
|
||||
MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws(?=/|$)", re.IGNORECASE)
|
||||
|
|
@ -36,6 +38,13 @@ def resolve_mantle_bearer_token(api_key: str | None) -> str | None:
|
|||
return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
|
||||
|
||||
|
||||
def split_mantle_region_prefix(model: str) -> tuple[str | None, str]:
|
||||
head, sep, tail = model.partition("/")
|
||||
if sep and head in BEDROCK_REGIONS:
|
||||
return head, tail
|
||||
return None, model
|
||||
|
||||
|
||||
def resolve_mantle_region(params: Mapping[str, object]) -> str:
|
||||
region: Final = params.get("aws_region_name")
|
||||
if isinstance(region, str) and region:
|
||||
|
|
@ -130,7 +139,7 @@ def mantle_supports_responses(model: str | None, model_cost: dict) -> bool:
|
|||
gpt-oss substring), so a substring gate would be wrong. A model absent from
|
||||
model_cost simply has no signal and returns False (chat-completions emulation).
|
||||
"""
|
||||
entry: Final = model_cost.get(f"bedrock_mantle/{model}", {})
|
||||
entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}", {}) if model else {}
|
||||
if "/v1/responses" in (entry.get("supported_endpoints") or []):
|
||||
return True
|
||||
return entry.get("mode") == "responses"
|
||||
|
|
@ -147,5 +156,5 @@ def mantle_base_segment(model: str | None, model_cost: dict) -> str:
|
|||
the base for the model's whole OpenAI-compatible surface, so both the chat and
|
||||
responses configs derive from it -- there is no separate model-name rule.
|
||||
"""
|
||||
entry: Final = model_cost.get(f"bedrock_mantle/{model}", {})
|
||||
entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}", {}) if model else {}
|
||||
return "openai/v1" if entry.get("use_openai_responses_path") is True else "v1"
|
||||
|
|
|
|||
|
|
@ -5890,6 +5890,81 @@ def test_resolve_vertex_location_for_cost_default_region(monkeypatch):
|
|||
assert _resolve("vertex_ai", None, None, "gemini-3.5-flash") == "us-central1"
|
||||
|
||||
|
||||
def test_resolve_mantle_region_for_cost(monkeypatch):
|
||||
"""Bedrock Mantle requests resolve the served region the way dispatch does (explicit
|
||||
aws_region_name, then the api_base host, then the default); other providers get None."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _resolve_mantle_region_for_cost
|
||||
|
||||
for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION_NAME", "AWS_REGION"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
assert _resolve_mantle_region_for_cost("bedrock", {"aws_region_name": "us-gov-west-1"}) is None
|
||||
assert _resolve_mantle_region_for_cost(None, {"aws_region_name": "us-gov-west-1"}) is None
|
||||
assert _resolve_mantle_region_for_cost("bedrock_mantle", {"aws_region_name": "us-gov-west-1"}) == "us-gov-west-1"
|
||||
assert (
|
||||
_resolve_mantle_region_for_cost(
|
||||
"bedrock_mantle",
|
||||
{"api_base": "https://bedrock-mantle.us-gov-west-1.api.aws/openai/v1/chat/completions"},
|
||||
)
|
||||
== "us-gov-west-1"
|
||||
)
|
||||
assert _resolve_mantle_region_for_cost("bedrock_mantle", None) == "us-east-1"
|
||||
|
||||
|
||||
def test_response_cost_calculator_prices_mantle_calls_on_the_served_region(monkeypatch):
|
||||
"""
|
||||
Mantle responses carry no region of their own (the OpenAI-compatible transform rebuilds the
|
||||
response, and streams never had one), so the logging layer must price them from the region
|
||||
the deployment was served in: an explicit aws_region_name or the api_base host, both of which
|
||||
must select the GovCloud row over the commercial one.
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
|
||||
for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION_NAME", "AWS_REGION"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
def cost_with(litellm_params):
|
||||
logging_obj = LitellmLogging(
|
||||
model="xai.grok-4.3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="mantle-region",
|
||||
function_id="f",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="xai.grok-4.3",
|
||||
user="",
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="resp-1",
|
||||
model="xai.grok-4.3",
|
||||
choices=[{"message": {"role": "assistant", "content": "hello"}, "index": 0, "finish_reason": "stop"}],
|
||||
usage={"prompt_tokens": 38, "completion_tokens": 20, "total_tokens": 58},
|
||||
)
|
||||
return logging_obj._response_cost_calculator(result=response)
|
||||
|
||||
commercial = litellm.model_cost["bedrock_mantle/xai.grok-4.3"]
|
||||
gov = litellm.model_cost["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
|
||||
expected_commercial = 38 * commercial["input_cost_per_token"] + 20 * commercial["output_cost_per_token"]
|
||||
expected_gov = 38 * gov["input_cost_per_token"] + 20 * gov["output_cost_per_token"]
|
||||
assert expected_gov != expected_commercial
|
||||
|
||||
assert cost_with({"api_base": ""}) == pytest.approx(expected_commercial)
|
||||
assert cost_with({"aws_region_name": "us-gov-west-1"}) == pytest.approx(expected_gov)
|
||||
assert cost_with(
|
||||
{"api_base": "https://bedrock-mantle.us-gov-west-1.api.aws/openai/v1/chat/completions"}
|
||||
) == pytest.approx(expected_gov)
|
||||
|
||||
|
||||
def test_response_cost_calculator_prices_proxy_vertex_calls_on_the_configured_location(monkeypatch):
|
||||
"""
|
||||
Proxy-shaped logging objects (created before the router picks a deployment) carry the
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.ht
|
|||
|
||||
import json
|
||||
import asyncio
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
import httpx
|
||||
|
|
@ -151,6 +151,27 @@ class TestBedrockMantleConfig:
|
|||
# /openai/v1 base per the AWS model card.
|
||||
assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1"
|
||||
|
||||
def test_region_prefixed_model_routes_to_that_region(self, monkeypatch, local_cost_map):
|
||||
for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION", "AWS_REGION_NAME"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
cfg = BedrockMantleChatConfig()
|
||||
api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model="us-gov-west-1/xai.grok-4.3")
|
||||
assert api_base == "https://bedrock-mantle.us-gov-west-1.api.aws/openai/v1"
|
||||
|
||||
def test_aws_region_name_param_beats_model_region_prefix(self, monkeypatch, local_cost_map):
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION", "AWS_REGION_NAME"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
cfg = BedrockMantleChatConfig()
|
||||
api_base, _ = cfg._get_openai_compatible_provider_info(
|
||||
None,
|
||||
None,
|
||||
litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1"),
|
||||
model="us-gov-west-1/xai.grok-4.3",
|
||||
)
|
||||
assert api_base == "https://bedrock-mantle.us-east-1.api.aws/openai/v1"
|
||||
|
||||
def test_default_api_base_fallback_to_us_east_1(self, monkeypatch):
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
|
|
@ -686,6 +707,120 @@ class TestBedrockMantleProviderResolution:
|
|||
assert model == "openai.gpt-oss-20b"
|
||||
|
||||
|
||||
def test_get_llm_provider_strips_region_prefix(self, monkeypatch, local_cost_map):
|
||||
for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION", "AWS_REGION_NAME"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
model, provider, _, api_base = litellm.get_llm_provider("bedrock_mantle/us-gov-west-1/xai.grok-4.3")
|
||||
assert provider == "bedrock_mantle"
|
||||
assert model == "xai.grok-4.3"
|
||||
assert api_base == "https://bedrock-mantle.us-gov-west-1.api.aws/openai/v1"
|
||||
|
||||
def test_completion_region_prefixed_model_sends_bare_model_to_that_region(self, monkeypatch, local_cost_map):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
for var in (
|
||||
"BEDROCK_MANTLE_API_KEY",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"BEDROCK_MANTLE_API_BASE",
|
||||
"BEDROCK_MANTLE_REGION",
|
||||
"AWS_REGION_NAME",
|
||||
"AWS_REGION",
|
||||
"AWS_PROFILE",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0")
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1733529600,
|
||||
"model": "xai.grok-4.3",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 38, "completion_tokens": 20, "total_tokens": 58},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
handler = Mock(side_effect=respond)
|
||||
response = litellm.completion(
|
||||
model="bedrock_mantle/us-gov-west-1/xai.grok-4.3",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
|
||||
)
|
||||
|
||||
gov = litellm.model_cost["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
|
||||
sent = handler.call_args.args[0]
|
||||
assert str(sent.url) == "https://bedrock-mantle.us-gov-west-1.api.aws/openai/v1/chat/completions"
|
||||
assert json.loads(sent.content)["model"] == "xai.grok-4.3"
|
||||
assert "/us-gov-west-1/bedrock/aws4_request" in sent.headers["Authorization"]
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
38 * gov["input_cost_per_token"] + 20 * gov["output_cost_per_token"]
|
||||
)
|
||||
|
||||
def test_responses_region_prefixed_model_prices_from_that_region_over_env_region(self, monkeypatch, local_cost_map):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
for var in (
|
||||
"BEDROCK_MANTLE_API_KEY",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"BEDROCK_MANTLE_API_BASE",
|
||||
"BEDROCK_MANTLE_REGION",
|
||||
"AWS_REGION",
|
||||
"AWS_PROFILE",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0")
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "resp_test",
|
||||
"object": "response",
|
||||
"created_at": 1733529600,
|
||||
"status": "completed",
|
||||
"model": "xai.grok-4.3",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_test",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"top_p": 1.0,
|
||||
"usage": {"input_tokens": 38, "output_tokens": 20, "total_tokens": 58},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
handler = Mock(side_effect=respond)
|
||||
response = litellm.responses(
|
||||
model="bedrock_mantle/us-gov-west-1/xai.grok-4.3",
|
||||
input="hello",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
|
||||
)
|
||||
|
||||
gov = litellm.model_cost["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
|
||||
sent = handler.call_args.args[0]
|
||||
assert str(sent.url) == "https://bedrock-mantle.us-gov-west-1.api.aws/openai/v1/responses"
|
||||
assert json.loads(sent.content)["model"] == "xai.grok-4.3"
|
||||
assert "/us-gov-west-1/bedrock/aws4_request" in sent.headers["Authorization"]
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
38 * gov["input_cost_per_token"] + 20 * gov["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
class TestBedrockMantlePricing:
|
||||
"""Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing."""
|
||||
|
||||
|
|
|
|||
|
|
@ -4313,6 +4313,59 @@ def test_select_model_name_applies_region_to_private_provider_response_model(_lo
|
|||
assert selected == "bedrock/us-east-1/anthropic.claude-v2:1"
|
||||
|
||||
|
||||
def test_completion_cost_region_name_prices_mantle_on_the_regional_row(_local_model_cost_map):
|
||||
"""completion_cost(region_name=...) must price a Bedrock Mantle call from the
|
||||
bedrock_mantle/<region>/<model> row when one exists, for the bare and the provider-prefixed
|
||||
model alike, and keep the flat row for regions without their own row."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model="xai.grok-4.3",
|
||||
usage={"prompt_tokens": 38, "completion_tokens": 20, "total_tokens": 58},
|
||||
)
|
||||
gov = litellm.model_cost["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
|
||||
flat = litellm.model_cost["bedrock_mantle/xai.grok-4.3"]
|
||||
expected_gov = 38 * gov["input_cost_per_token"] + 20 * gov["output_cost_per_token"]
|
||||
expected_flat = 38 * flat["input_cost_per_token"] + 20 * flat["output_cost_per_token"]
|
||||
assert expected_gov != expected_flat
|
||||
|
||||
for model in ("xai.grok-4.3", "bedrock_mantle/xai.grok-4.3"):
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
region_name="us-gov-west-1",
|
||||
) == pytest.approx(expected_gov)
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
region_name="eu-west-1",
|
||||
) == pytest.approx(expected_flat)
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response, model="xai.grok-4.3", custom_llm_provider="bedrock_mantle"
|
||||
) == pytest.approx(expected_flat)
|
||||
|
||||
|
||||
def test_cost_per_token_region_name_applies_to_provider_prefixed_model(_local_model_cost_map):
|
||||
"""A provider-prefixed model must still find its bedrock_mantle/<region>/<model> row instead of
|
||||
composing the region key with the provider segment twice."""
|
||||
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="bedrock_mantle/xai.grok-4.3",
|
||||
prompt_tokens=38,
|
||||
completion_tokens=20,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
region_name="us-gov-west-1",
|
||||
)
|
||||
gov = litellm.model_cost["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
|
||||
|
||||
assert prompt_cost + completion_cost == pytest.approx(
|
||||
38 * gov["input_cost_per_token"] + 20 * gov["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_select_model_name_keeps_base_model_free_of_region(_local_model_cost_map):
|
||||
"""An explicit base_model keeps pricing on that model's own key even when the request carries a
|
||||
region with different regional rates, so the private provider model never widens region pricing."""
|
||||
|
|
@ -4329,6 +4382,29 @@ def test_select_model_name_keeps_base_model_free_of_region(_local_model_cost_map
|
|||
assert selected == "bedrock/moonshotai.kimi-k2.5"
|
||||
|
||||
|
||||
def test_completion_cost_base_model_ignores_regional_row(_local_model_cost_map):
|
||||
"""A deployment with base_model set is priced from that model's own row even when the response
|
||||
carries a region whose regional row charges different rates."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model="my-bedrock-deployment",
|
||||
usage={"prompt_tokens": 1000, "completion_tokens": 0, "total_tokens": 1000},
|
||||
)
|
||||
response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "eu-central-1"}
|
||||
flat = litellm.model_cost["anthropic.claude-instant-v1"]
|
||||
regional = litellm.model_cost["bedrock/eu-central-1/anthropic.claude-instant-v1"]
|
||||
assert flat["input_cost_per_token"] != regional["input_cost_per_token"]
|
||||
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="my-bedrock-deployment",
|
||||
custom_llm_provider="bedrock",
|
||||
base_model="anthropic.claude-instant-v1",
|
||||
) == pytest.approx(1000 * flat["input_cost_per_token"])
|
||||
|
||||
|
||||
def test_completion_cost_nonzero_for_slash_alias_model_name(_local_model_cost_map):
|
||||
"""End-to-end cost through a "/"-containing alias must price above zero (#38069)."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue