diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0b91e57f01e..e9f9df90d4b 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -44,7 +44,7 @@ from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncreme from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget -from litellm.types.router import DeploymentTypedDict +from litellm.types.router import DeploymentTypedDict, ModelGroupInfo @dataclass @@ -705,14 +705,7 @@ def _model_access_group_counter(group: str, budget: ModelAccessGroupBudget | Non def _dedupe_tags(tags: list[str]) -> list[str]: - seen: Final = set() - deduped_tags: Final = [] - for tag in tags: - if tag in seen: - continue - seen.add(tag) - deduped_tags.append(tag) - return deduped_tags + return list(dict.fromkeys(tags)) @with_service_target(AUTH_OBJECTS_TARGET) @@ -1506,6 +1499,9 @@ def _estimate_image_generation_cost( return cost_per_image * max(n, 1) +_DECLARED_GROUP_FIELDS: Final = MappingProxyType(dict.fromkeys(ModelGroupInfo.model_fields, True)) + + def _get_model_cost_info( model: str, llm_router: Router | None, @@ -1513,7 +1509,7 @@ def _get_model_cost_info( if llm_router is not None: model_group_info: Final = llm_router.cached_model_group_info(model) if model_group_info is not None: - return model_group_info.model_dump() + return model_group_info.model_dump(include=_DECLARED_GROUP_FIELDS) return dict(litellm.get_model_info(model=model)) diff --git a/litellm/router.py b/litellm/router.py index 2d582699561..93b7ab7e08f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -143,6 +143,7 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES from litellm.litellm_core_utils.ptu_pricing import ( PTU_COST_ATTRIBUTION_ENV_VAR, @@ -450,6 +451,7 @@ from litellm.types.router import ( SearchToolTypedDict, TaggedPreRoutingStrategy, holds_secret_pointer, + is_usd_price_key, model_info_prices, price_as_float, ) @@ -545,12 +547,6 @@ RouterStrategySelector: TypeAlias = ( ) -def _deployment_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: - cost_map_key: Final = model_info.get("key") - entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None - return model_info_prices(model_info if entry is None else {**entry, **model_info}) - - def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: """Single owner of the environment-gating rule: a deployment whose model_info names `supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list. @@ -719,6 +715,28 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, object]: + configured_info: Final = _configured_model_info(deployment) + model_id: Final = configured_info.get("id") + custom_priced: Final = use_custom_pricing_for_model( + { + **_LITELLM_PARAMS_ADAPTER.validate_python(deployment.get("litellm_params") or {}), + "metadata": {"model_info": dict(configured_info)}, + } + ) + registered_entry: Final = raw_cost_map_entry(model_id) if custom_priced and isinstance(model_id, str) else None + if registered_entry is not None: + return registered_entry + cost_map_key: Final = model_info.get("key") + catalog_entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None + return model_info if catalog_entry is None else {**catalog_entry, **model_info} + + +def _deployment_prices(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, float]: + token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")} + return model_info_prices({**_billed_pricing_entry(deployment, model_info), **token_prices}) + + def _every_deployment_refused( deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...] ) -> bool: @@ -12702,7 +12720,7 @@ class Router: deployment_prices = ( model_info_prices(_configured_model_info(model)) if model_info is None - else _deployment_prices(model_info) + else _deployment_prices(model, model_info) ) # get llm provider @@ -12740,7 +12758,7 @@ class Router: "model_group": user_facing_model_group_name, "providers": [llm_provider], "description": model_group_description, - **model_info, + **{key: value for key, value in model_info.items() if not is_usd_price_key(key)}, **deployment_prices, "supports_fast_mode": True, "supported_reasoning_efforts": None, diff --git a/litellm/types/router.py b/litellm/types/router.py index 183e12a4f4c..ac60e333ccc 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -7,6 +7,7 @@ import enum import math from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from functools import lru_cache from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -1000,12 +1001,17 @@ def _outprices(price: float, listed: object) -> bool: return listed_price is None or price > listed_price -def _is_billed_price_field(key: str) -> bool: - return key in _GetModelCostMap.loaded_model_cost_map_fields() or _is_server_derived_pricing_key(key) +@lru_cache(maxsize=4096) +def _is_usd_price_key_in(key: str, catalog_fields: frozenset[str]) -> bool: + return ( + ("cost_per_" in key or "token_cost" in key) + and "_dbu_" not in key + and (key in catalog_fields or _is_server_derived_pricing_key(key)) + ) def is_usd_price_key(key: str) -> bool: - return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key and _is_billed_price_field(key) + return _is_usd_price_key_in(key, _GetModelCostMap.loaded_model_cost_map_fields()) def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index c4cde058dcd..8ace1f83439 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -176,6 +176,7 @@ ignored_function_names = [ "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py "_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py + "_billed_pricing_entry", # Read by _deployment_prices, tested through the custom-priced model group info test in test_router.py "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py "_drops_params", # Read by split_request_priority and priority_header_update, tested through both "_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 024fe5229be..50ca8fcacd5 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -220,6 +220,54 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) +def _priced_group_router(extra_prices: dict[str, float], mode: str) -> Router: + return Router( + model_list=[ + { + "model_name": "priced-group", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-fake", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + **extra_prices, + }, + "model_info": {"id": "priced-deployment", "mode": mode}, + } + ] + ) + + +@pytest.mark.parametrize( + "extra_prices, mode, route, body", + [ + ( + {"output_cost_per_reasoning_token": 1e-04}, + "chat", + "/chat/completions", + {"model": "priced-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + ), + ( + {"output_cost_per_image": 0.5}, + "image_generation", + "/images/generations", + {"model": "priced-group", "prompt": "a cat", "n": 4}, + ), + ], +) +def test_prices_a_group_lists_leave_its_budget_reservation_unchanged( + extra_prices: dict[str, float], mode: str, route: str, body: dict[str, object] +) -> None: + without_extra: Final = estimate_request_max_cost( + request_body=body, route=route, llm_router=_priced_group_router({}, mode) + ) + with_extra: Final = estimate_request_max_cost( + request_body=body, route=route, llm_router=_priced_group_router(extra_prices, mode) + ) + + assert with_extra == without_extra + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index ad4b677ec10..760e8f6b34b 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -97,7 +97,7 @@ from litellm.llms.base_llm.vector_store.transformation import( LiteLLMVectorStoreEmbeddingExecutor, RouterVectorStoreEmbeddingExecutor, ) -from litellm.types.utils import CallTypes, CredentialItem, Usage +from litellm.types.utils import CallTypes, CredentialItem, ModelResponse, Usage from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE @@ -2672,6 +2672,59 @@ def test_model_group_info_never_takes_prices_from_a_cost_map_entry_named_like_th assert reported.keys().isdisjoint({"input_cost_per_token_priority", "output_cost_per_token_batches"}) +def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_bills( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.utils import _invalidate_model_cost_lowercase_map + + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + "made-up-long-context-model": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "custom-priced-group", + "litellm_params": { + "model": "openai/made-up-long-context-model", + "api_key": "fake", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, + "model_info": {"id": "custom-priced-deployment"}, + } + ] + ) + long_prompt_tokens: Final = 300_000 + + response: Final = router.completion( + model="custom-priced-group", + messages=[{"role": "user", "content": "hi"}], + mock_response=ModelResponse( + model="made-up-long-context-model", + choices=[{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop", "index": 0}], + usage=Usage(prompt_tokens=long_prompt_tokens, completion_tokens=0, total_tokens=long_prompt_tokens), + ), + ) + info: Final = router.get_model_group_info(model_group="custom-priced-group") + + assert info is not None + listed: Final = info.model_dump() + assert response._hidden_params["response_cost"] == pytest.approx(long_prompt_tokens * listed["input_cost_per_token"]) + assert "input_cost_per_token_above_200k_tokens" not in listed + + def test_model_group_info_still_validates_from_attributes(): source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)