From 555a65f1ca93466ef061fdee6f366398872cf0d0 Mon Sep 17 00:00:00 2001 From: Sirine Jnayeh Date: Wed, 28 Oct 2026 12:01:50 +0400 Subject: [PATCH] fix(types): accept every cost map row in the Bedrock profile pricing merge A Bedrock deployment with model_id and a base_model runs the backend model's cost map row through _PricingModelInfo since #47419. Two fields of ModelInfoBase were stricter than the rows the cost map ships, so validation failed and the router dropped the deployment at startup mode had no rerank, moderation, image_edit, video_generation, search, vector_store, guardrail or container, which the cost map schema already allows. provider_specific_entry was typed dict[str, float] while 38 rows store strings or nested objects there, among them amazon.titan-embed-text-v2:0. The geo and speed multipliers now only read numeric values from it Fixes #48545 --- .../litellm_core_utils/llm_cost_calc/utils.py | 13 +++++---- litellm/types/utils.py | 10 ++++++- .../unit/test_router_model_cost_isolation.py | 23 ++++++++++++++++ tests/unit/test_utils.py | 27 +++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3b49f239afe..3f0208cfa08 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1396,15 +1396,18 @@ def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> inference_geo: Final = getattr(usage, "inference_geo", None) if not isinstance(inference_geo, str) or inference_geo.lower() in ("global", "not_available"): return 1.0 - provider_specific_entry: Final[dict[str, float]] = model_info.get("provider_specific_entry") or {} - return float(provider_specific_entry.get(inference_geo.lower(), 1.0)) + return _provider_specific_multiplier(model_info, inference_geo.lower()) def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) -> float: - provider_specific_entry: Final = model_info.get("provider_specific_entry") - if getattr(usage, "speed", None) != "fast" or not provider_specific_entry: + if getattr(usage, "speed", None) != "fast": return 1.0 - return float(provider_specific_entry.get("fast", 1.0)) + return _provider_specific_multiplier(model_info, "fast") + + +def _provider_specific_multiplier(model_info: ModelInfo, name: str) -> float: + multiplier: Final = (model_info.get("provider_specific_entry") or {}).get(name) + return float(multiplier) if isinstance(multiplier, int | float) else 1.0 def _resolve_reasoning_token_cost( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2869f9b2577..c33aff61259 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -477,13 +477,21 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "extraction", "ocr", "realtime", + "rerank", + "moderation", + "image_edit", + "video_generation", + "search", + "vector_store", + "guardrail", + "container", ] ] supported_endpoints: list[str] | None use_openai_responses_path: bool | None tpm: int | None rpm: int | None - provider_specific_entry: dict[str, float] | None + provider_specific_entry: dict[str, object] | None uses_embed_content: bool | None diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..f65137287cf 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -948,6 +948,29 @@ def test_bedrock_deployment_inherits_invoked_profile_around_custom_prices( assert litellm.model_cost[profile] == profile_prices +@pytest.mark.parametrize( + ("base_model", "custom_price"), + (("cohere.rerank-v3-5:0", "input_cost_per_query"), ("amazon.titan-embed-text-v2:0", "input_cost_per_token")), +) +def test_bedrock_profile_deployment_with_a_non_chat_base_model_is_registered(base_model: str, custom_price: str) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "profile-non-chat", + "litellm_params": { + "model": f"bedrock/arn:aws:bedrock:eu-central-1::foundation-model/{base_model}", + "model_id": "arn:aws:bedrock:eu-central-1:123456789012:application-inference-profile/abc123", + "aws_region_name": "eu-central-1", + }, + "model_info": {"id": "profile-non-chat-deployment", "base_model": base_model, custom_price: 0.00175}, + } + ] + ) + + assert router.has_model_id("profile-non-chat-deployment") + assert litellm.model_cost["profile-non-chat-deployment"][custom_price] == 0.00175 + + @pytest.mark.parametrize("catalog_entry", (None, {"litellm_provider": 1, "mode": "chat"})) @pytest.mark.parametrize("known_model", (False, True)) @pytest.mark.parametrize("explicit_base", (False, True)) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38287b25169..cf723530a70 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1426,6 +1426,33 @@ def test_cost_map_schema_accepts_12hr_cache_write_rates_for_batch_and_ultrafast_ assert Draft7Validator(_COST_MAP_ROW_SCHEMA).is_valid({"synthetic-12hr-cache-write-model": row}) +def test_pricing_overrides_accept_every_row_the_cost_map_ships() -> None: + from litellm.utils import ( + ModelInfoLookupMiss, + lookup_model_info_for_registration, + model_info_with_pricing_overrides, + ) + + shipped: Final = json.loads((Path(__file__).parents[2] / "model_prices_and_context_window.json").read_text()) + rows: Final = { + model: row + for model, row in shipped.items() + if model not in ("sample_spec", "fallback_generalizations") and isinstance(row, dict) and "mode" in row + } + looked_up: Final = { + model: lookup_model_info_for_registration(model=model, custom_llm_provider=row.get("litellm_provider")) + for model, row in rows.items() + } + found: Final = {model: info for model, info in looked_up.items() if not isinstance(info, ModelInfoLookupMiss)} + priced: Final = { + model: model_info_with_pricing_overrides(info, {"input_cost_per_query": 0.5})["mode"] + for model, info in found.items() + } + + assert priced == {model: info["mode"] for model, info in found.items()} + assert len(found) > len(rows) // 2, f"only {len(found)} of {len(rows)} cost map rows resolved" + + def test_max_tokens_consistency(): """ Test that max_tokens == max_output_tokens for all models.