diff --git a/litellm/router.py b/litellm/router.py index 047a8762097..1f662c1e27a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12638,6 +12638,11 @@ class Router: model_info = None deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict) + deployment_prices = ( + model_info_prices(_configured_model_info(model)) + if model_info is None + else _deployment_prices(model_info) + ) # get llm provider litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) @@ -12675,7 +12680,7 @@ class Router: "providers": [llm_provider], "description": model_group_description, **model_info, - **_deployment_prices(model_info), + **deployment_prices, "supports_fast_mode": True, "supported_reasoning_efforts": None, "supported_endpoints": None, @@ -12709,7 +12714,7 @@ class Router: ) ): model_group_info.max_output_tokens = model_info["max_output_tokens"] - model_group_info = model_group_info.with_higher_prices(_deployment_prices(model_info)) + model_group_info = model_group_info.with_higher_prices(deployment_prices) if ( model_info.get("supports_parallel_function_calling", None) is not None and model_info["supports_parallel_function_calling"] is True diff --git a/litellm/types/router.py b/litellm/types/router.py index bc2fbfc7e31..0957708ecf9 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -41,6 +41,7 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap as _GetModelCostMap from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router_weights import RouterWeights @@ -71,6 +72,9 @@ from .utils import ( ModelResponse, StandardLoggingRoutingDecision, ) +from .utils import ( + is_server_derived_pricing_key as _is_server_derived_pricing_key, +) from .utils import ( server_owned_wif_litellm_params as _server_owned_wif_litellm_params, ) @@ -992,8 +996,12 @@ def _outprices(price: float, listed: object) -> bool: return listed_price is None or price > listed_price +def _is_billed_price_field(key: str) -> bool: + return key in _GetModelCostMap.loaded_model_cost_map_fields() or _is_server_derived_pricing_key(key) + + def is_usd_price_key(key: str) -> bool: - return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key + return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key and _is_billed_price_field(key) def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index ddf690b022d..bdda58f222a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2509,6 +2509,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_999k_tokens": 9e-06, }, ) router: Final = litellm.Router( @@ -2525,6 +2526,9 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_dbu_cost_per_token": 7e-07, "cost_center": 4021, "cost_allocation_pct": "30", + "internal_chargeback_cost_per_token": 0.5, + "team_cost_per_seat": 40, + "input_cost_per_tokens": 9e-06, }, } for index in deployment_order @@ -2541,9 +2545,13 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_999k_tokens": 9e-06, } assert reported.keys().isdisjoint( { + "internal_chargeback_cost_per_token", + "team_cost_per_seat", + "input_cost_per_tokens", "output_cost_per_video_per_second", "output_cost_per_second", "search_context_cost_per_query", @@ -2596,6 +2604,44 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon assert "input_dbu_cost_per_token" not in reported +def test_model_group_info_never_takes_prices_from_a_cost_map_entry_named_like_the_group( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.utils import _invalidate_model_cost_lowercase_map + + group_name: Final = "made-up-group-named-like-a-catalog-model" + router: Final = litellm.Router( + model_list=[ + { + "model_name": group_name, + "litellm_params": {"model": "gemini/made-up-unmapped-model", "api_key": "fake"}, + "model_info": {"id": "self-hosted-deployment", "input_cost_per_token": 1e-07}, + } + ] + ) + monkeypatch.setattr( + litellm, + "model_cost", + { + group_name: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 5e-06, + "input_cost_per_token_priority": 1e-05, + "output_cost_per_token_batches": 7e-06, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + + info: Final = router.get_model_group_info(model_group=group_name) + + assert info is not None + reported: Final = info.model_dump() + assert reported["input_cost_per_token"] == 1e-07 + assert reported.keys().isdisjoint({"input_cost_per_token_priority", "output_cost_per_token_batches"}) + + def test_model_group_info_still_validates_from_attributes(): source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)