diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..edfabf43ad0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -910,7 +910,9 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: return 0.0 -def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None: +def get_cost_per_unit( + model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0 +) -> float | None: # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml) cost_per_unit: Final = model_info.get(cost_key) if isinstance(cost_per_unit, float): @@ -954,6 +956,19 @@ def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float _get_cost_per_unit = get_cost_per_unit +_HALF_PRICED_BATCH_KEYS: Final = frozenset( + {f"input_cost_per_token{_BATCH_KEY_SUFFIX}", f"output_cost_per_token{_BATCH_KEY_SUFFIX}"} +) + + +def billed_rate(model_info: Mapping[str, object], cost_key: str) -> float | None: + rate: Final = get_cost_per_unit(model_info, cost_key, default_value=None) + if rate is not None or cost_key not in _HALF_PRICED_BATCH_KEYS: + return rate + standard_rate: Final = get_cost_per_unit(model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None) + return None if standard_rate is None else standard_rate / 2 + + def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None: """The prices a deployment sets itself, as floats; None when it sets none that parse.""" if model_info is None: diff --git a/litellm/router.py b/litellm/router.py index 9d7017aa0de..ec713e55938 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -144,7 +144,7 @@ from litellm.litellm_core_utils.hidden_params import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model -from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES, billed_rate from litellm.litellm_core_utils.ptu_pricing import ( PTU_COST_ATTRIBUTION_ENV_VAR, declares_ptu, @@ -732,9 +732,38 @@ def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[ return model_info if catalog_entry is None else {**catalog_entry, **model_info} -def _deployment_prices(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, float]: +def _deployment_price_entry( + deployment: Mapping[str, object], model_info: Mapping[str, object] | None +) -> Mapping[str, object]: + if model_info is None: + return _configured_model_info(deployment) token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")} - return model_info_prices({**_billed_pricing_entry(deployment, model_info), **token_prices}) + return {**_billed_pricing_entry(deployment, model_info), **token_prices} + + +@dataclass(frozen=True, slots=True) +class _GroupDeployment: + model_info: ModelMapInfo | None + price_entry: Mapping[str, object] + team_may_use: bool + + def listed_prices(self) -> Mapping[str, float]: + prices: Final = model_info_prices(self.price_entry) + if self.team_may_use: + return prices + return MappingProxyType({key: price for key, price in prices.items() if key in ModelGroupInfo.model_fields}) + + +def _highest_billed_rate(cost_key: str, price_entries: Sequence[Mapping[str, object]]) -> float | None: + rates: Final = (price_as_float(billed_rate(entry, cost_key)) for entry in price_entries) + return max((rate for rate in rates if rate is not None), default=None) + + +def _highest_billed_prices( + listed_keys: Iterable[str], price_entries: Sequence[Mapping[str, object]] +) -> Mapping[str, float]: + rates: Final = ((key, _highest_billed_rate(key, price_entries)) for key in listed_keys) + return MappingProxyType({key: rate for key, rate in rates if rate is not None}) def _every_deployment_refused( @@ -12652,17 +12681,12 @@ class Router: (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, ) - for model in model_list: - is_match = False - if ( - "model_name" in model - and model["model_name"] == model_group - or "model_name" in model - and self.pattern_router.route(model_group) is not None - ): # exact match - is_match = True - - if not is_match: + group_deployments: Final = tuple( + self._group_deployment(model, team_id) if self._is_model_group_member(model, model_group) else None + for model in model_list + ) + for model, group_deployment in zip(model_list, group_deployments, strict=True): + if group_deployment is None: continue # model in model group found # litellm_params = LiteLLM_Params(**model["litellm_params"]) @@ -12703,22 +12727,9 @@ class Router: if _deployment_otpm is None: _deployment_otpm = model_info_dict.get("otpm", None) - # get model info - try: - model_id = model_info_dict.get("id", None) - if model_id is not None: - model_info = self.get_deployment_model_info(model_id=model_id, model_name=litellm_params.model) - else: - model_info = None - except Exception: - model_info = None - + model_info = group_deployment.model_info deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict) - deployment_prices = ( - model_info_prices(_configured_model_info(model)) - if model_info is None - else _deployment_prices(model, model_info) - ) + deployment_prices = group_deployment.listed_prices() # get llm provider litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) @@ -12835,7 +12846,7 @@ class Router: deployment_supported_endpoints( model_info.get("supported_endpoints"), model, llm_provider ) - if team_may_use_deployment(_configured_model_info(model).get("team_id"), team_id) + if group_deployment.team_may_use else () ), ) @@ -12899,7 +12910,39 @@ class Router: if configurable_clientside_auth_params is not None: model_group_info.configurable_clientside_auth_params = configurable_clientside_auth_params - return model_group_info + usable_price_entries: Final = tuple( + deployment.price_entry + for deployment in group_deployments + if deployment is not None and deployment.team_may_use + ) + return model_group_info.with_higher_prices( + _highest_billed_prices(model_info_prices(model_group_info.model_dump()), usable_price_entries) + ) + + return None + + def _is_model_group_member(self, deployment: DeploymentTypedDict, model_group: str) -> bool: + return "model_name" in deployment and ( + deployment["model_name"] == model_group or self.pattern_router.route(model_group) is not None + ) + + def _deployment_model_info(self, deployment: DeploymentTypedDict) -> ModelMapInfo | None: + model_id: Final = _configured_model_info(deployment).get("id") + model_name: Final = deployment["litellm_params"].get("model") + if not isinstance(model_id, str) or model_name is None: + return None + try: + return self.get_deployment_model_info(model_id=model_id, model_name=model_name) + except Exception: + return None + + def _group_deployment(self, deployment: DeploymentTypedDict, team_id: str | None) -> _GroupDeployment: + model_info: Final = self._deployment_model_info(deployment) + return _GroupDeployment( + model_info=model_info, + price_entry=_deployment_price_entry(deployment, model_info), + team_may_use=team_may_use_deployment(_configured_model_info(deployment).get("team_id"), team_id), + ) def get_model_group_info( self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 8ace1f83439..194e99f4a9b 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -175,8 +175,14 @@ ignored_function_names = [ "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py - "_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py - "_billed_pricing_entry", # Read by _deployment_prices, tested through the custom-priced model group info test in test_router.py + "_deployment_price_entry", # Read by _group_deployment, tested through the cost map model group info test in test_router.py + "_billed_pricing_entry", # Read by _deployment_price_entry, tested through the custom-priced model group info test in test_router.py + "listed_prices", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py + "_highest_billed_rate", # Read by _highest_billed_prices, tested through the tier and batch model group info test in test_router.py + "_highest_billed_prices", # Read by _set_model_group_info, tested through the tier and batch model group info test in test_router.py + "_is_model_group_member", # Read by _set_model_group_info, tested through every model group info test in test_router.py + "_deployment_model_info", # Read by _group_deployment, tested through the cost map model group info test in test_router.py + "_group_deployment", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py "_drops_params", # Read by split_request_priority and priority_header_update, tested through both "_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 760e8f6b34b..05166b1c1a2 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2725,6 +2725,128 @@ def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_ assert "input_cost_per_token_above_200k_tokens" not in listed +def test_model_group_info_lists_tier_and_batch_prices_at_the_most_any_deployment_bills( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.cost_calculator import batch_cost_calculator + from litellm.utils import _invalidate_model_cost_lowercase_map + + backends: Final = ("made-up-tiered-backend", "made-up-untiered-backend") + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + backends[0]: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "input_cost_per_token_flex": 5e-07, + "output_cost_per_token_flex": 2e-06, + "input_cost_per_token_batches": 4e-07, + "output_cost_per_token_batches": 2e-06, + }, + backends[1]: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.2e-05, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "mixed-backend-group", + "litellm_params": {"model": f"openai/{backend}", "api_key": "fake"}, + "model_info": {"id": backend}, + } + for backend in backends + ] + ) + tokens: Final = 1_000_000 + usage: Final = Usage(prompt_tokens=tokens, completion_tokens=tokens, total_tokens=2 * tokens) + flex_bills: Final = tuple( + litellm.cost_per_token( + model=backend, + custom_llm_provider="openai", + prompt_tokens=tokens, + completion_tokens=tokens, + service_tier="flex", + ) + for backend in backends + ) + batch_bills: Final = tuple( + batch_cost_calculator(usage, model=backend, custom_llm_provider="openai") for backend in backends + ) + highest_bills: Final = ( + max(prompt_cost for prompt_cost, _ in flex_bills), + max(completion_cost for _, completion_cost in flex_bills), + max(prompt_cost for prompt_cost, _ in batch_bills), + max(completion_cost for _, completion_cost in batch_bills), + ) + + info: Final = router.get_model_group_info(model_group="mixed-backend-group") + + assert info is not None + listed: Final = info.model_dump() + listed_keys: Final = ( + "input_cost_per_token_flex", + "output_cost_per_token_flex", + "input_cost_per_token_batches", + "output_cost_per_token_batches", + ) + assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills) + + +@pytest.mark.parametrize(("team_id", "listed_cache_read_cost"), [(None, 1e-07), ("team-b", 1e-07), ("team-a", 9e-06)]) +def test_model_group_info_lists_a_team_deployment_price_only_to_that_team( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, listed_cache_read_cost: float +): + from litellm.utils import _invalidate_model_cost_lowercase_map + + cache_read_costs: Final = {"made-up-shared-backend": 1e-07, "made-up-team-backend": 9e-06} + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + **{ + backend: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": cache_read_cost, + } + for backend, cache_read_cost in cache_read_costs.items() + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-scoped-group", + "litellm_params": {"model": "openai/made-up-shared-backend", "api_key": "fake"}, + "model_info": {"id": "shared-deployment"}, + }, + { + "model_name": "team-scoped-group", + "litellm_params": {"model": "openai/made-up-team-backend", "api_key": "fake"}, + "model_info": {"id": "team-a-deployment", "team_id": "team-a"}, + }, + ] + ) + + info: Final = router.get_model_group_info(model_group="team-scoped-group", team_id=team_id) + + assert info is not None + assert info.model_dump().get("cache_read_input_token_cost") == listed_cache_read_cost + + def test_model_group_info_still_validates_from_attributes(): source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)