From daf08f1f207bb912e1ff5ca046839efd1cfa8d17 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:23:59 -0700 Subject: [PATCH] fix(router): list only USD price keys on model group info, show token prices per 1M tokens, and read each deployment's prices in one pass --- litellm/router.py | 8 ++--- litellm/types/router.py | 35 ++++++++++--------- .../router_code_coverage.py | 2 +- .../proxy_server/test_routes_model_info.py | 3 ++ tests/unit/test_router/test_router.py | 18 +++++++--- .../AIHub/ModelExtraPrices.test.tsx | 8 +++-- .../src/components/AIHub/ModelExtraPrices.tsx | 16 +++++++-- 7 files changed, 61 insertions(+), 29 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 9e8f958ede8..126c7a5a82c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -511,10 +511,10 @@ RouterStrategySelector: TypeAlias = ( ) -def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: +def _deployment_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: cost_map_key: Final = model_info.get("key") entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None - return model_info_prices(entry) if entry is not None else MappingProxyType({}) + return model_info_prices(model_info if entry is None else {**entry, **model_info}) def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: @@ -12592,8 +12592,8 @@ class Router: "model_group": user_facing_model_group_name, "providers": [llm_provider], "description": model_group_description, - **_cost_map_prices(model_info), **model_info, + **_deployment_prices(model_info), "supports_fast_mode": True, "supported_reasoning_efforts": None, "supported_endpoints": None, @@ -12627,7 +12627,7 @@ class Router: ) ): model_group_info.max_output_tokens = model_info["max_output_tokens"] - model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info}) + model_group_info = model_group_info.with_higher_prices(_deployment_prices(model_info)) if ( model_info.get("supports_parallel_function_calling", None) is not None and model_info["supports_parallel_function_calling"] is True diff --git a/litellm/types/router.py b/litellm/types/router.py index 48434abd35e..bc2fbfc7e31 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -982,13 +982,22 @@ def price_as_float(value: object) -> float | None: return None try: price: Final = float(value) - except ValueError: + except (ValueError, OverflowError): return None return price if math.isfinite(price) else None +def _outprices(price: float, listed: object) -> bool: + listed_price: Final = price_as_float(listed) + return listed_price is None or price > listed_price + + +def is_usd_price_key(key: str) -> bool: + return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key + + def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: - candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key) + candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if is_usd_price_key(key)) return MappingProxyType({key: price for key, price in candidates if price is not None}) @@ -1042,22 +1051,16 @@ class ModelGroupInfo(LiteLLMBaseModel): if not isinstance(data, Mapping): return data fields: Final = _RAW_FIELDS.validate_python(data) - declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key} + field_names: Final = cls.model_fields.keys() + declared: Final = { + key: value for key, value in fields.items() if key in field_names and not is_usd_price_key(key) + } return {**declared, **model_info_prices(fields)} - @property - def prices(self) -> Mapping[str, float]: - return model_info_prices({**vars(self), **self.__pydantic_extra__}) - - def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo": - current: Final = self.prices - return self.model_copy( - update={ - key: price - for key, price in model_info_prices(model_info).items() - if key not in current or price > current[key] - } - ) + def with_higher_prices(self, prices: Mapping[str, float]) -> "ModelGroupInfo": + listed: Final[Mapping[str, object]] = {**vars(self), **self.__pydantic_extra__} + higher: Final = {key: price for key, price in prices.items() if _outprices(price, listed.get(key))} + return self.model_copy(update=higher) if higher else self class AssistantsTypedDict(TypedDict): diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 19481b8a161..fa6829dc183 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -172,7 +172,7 @@ ignored_function_names = [ "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py - "_cost_map_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py + "_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py ] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index a2ad764dfa6..68fb8f15abd 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -1016,6 +1016,8 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep "output_cost_per_image_2K": "Infinity", "input_cost_per_token": "NaN", "custom_tags": ["images"], + "cost_center": 4021, + "input_dbu_cost_per_token": 7e-07, }, }, { @@ -1057,6 +1059,7 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep "output_cost_per_image_2K": None, } assert group["input_cost_per_token"] is None + assert (group.get("cost_center"), group.get("input_dbu_cost_per_token")) == (None, None) def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a0f29dd3e59..94b60ce2ab2 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2509,7 +2509,6 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, - "input_dbu_cost_per_token": 7e-07, }, ) router: Final = litellm.Router( @@ -2523,6 +2522,9 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "output_cost_per_video_per_second": "free", "output_cost_per_second": "inf", "search_context_cost_per_query": {"search_context_size_low": 0.01}, + "input_dbu_cost_per_token": 7e-07, + "cost_center": 4021, + "cost_allocation_pct": "30", }, } for index in deployment_order @@ -2539,13 +2541,15 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_batches": 1e-06, - "input_dbu_cost_per_token": 7e-07, } assert reported.keys().isdisjoint( { "output_cost_per_video_per_second", "output_cost_per_second", "search_context_cost_per_query", + "input_dbu_cost_per_token", + "cost_center", + "cost_allocation_pct", "litellm_provider", "key", "id", @@ -2560,14 +2564,18 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon "output_cost_per_image": 0.08, "output_cost_per_image_0.5K": 0.06, "output_cost_per_image_4K": 0.16, - "input_dbu_cost_per_token": 0.0002, } monkeypatch.setattr( litellm, "model_cost", { **litellm.model_cost, - "made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices}, + "made-up-resolution-model": { + "litellm_provider": "openai", + "mode": "image_generation", + **cost_map_prices, + "input_dbu_cost_per_token": 0.0002, + }, }, ) _invalidate_model_cost_lowercase_map() @@ -2585,6 +2593,7 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon assert info is not None reported: Final = info.model_dump() assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices + assert "input_dbu_cost_per_token" not in reported def test_model_group_info_still_validates_from_attributes(): @@ -2945,6 +2954,7 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto ("Infinity", None), ("-inf", None), (float("nan"), None), + (10**400, None), ], ) def test_price_as_float(value: object, expected: float | None): diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx index 97934392993..622ad6c9e21 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { ModelExtraPrices } from "./ModelExtraPrices"; describe("ModelExtraPrices", () => { - it("lists every numeric price besides the per-token pair, sorted by name", () => { + it("lists every USD price besides the per-token pair, sorted by name, with token prices per 1M tokens", () => { render( { output_cost_per_image: 0.04, input_cost_per_token_priority: 0.0000025, output_cost_per_image_4K: 0.16, + cache_read_input_token_cost: 0.0000005, + input_dbu_cost_per_token: 0.000142858, + cost_center: 4021, output_cost_per_video_per_second: "free", search_context_cost_per_query: { search_context_size_low: 0.01 }, max_input_tokens: 128000, @@ -22,7 +25,8 @@ describe("ModelExtraPrices", () => { const terms = screen.getAllByRole("term").map((term) => term.textContent); const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent); expect(terms.map((term, index) => [term, definitions[index]])).toEqual([ - ["input_cost_per_token_priority", "$0.0000025"], + ["cache_read_input_token_cost", "$0.5 per 1M tokens"], + ["input_cost_per_token_priority", "$2.5 per 1M tokens"], ["output_cost_per_image", "$0.04"], ["output_cost_per_image_4K", "$0.16"], ]); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx index bb24dd5e115..cbecc76149e 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx @@ -1,14 +1,26 @@ const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]); +const TOKENS_PER_LISTED_PRICE = 1_000_000; + const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 }); -const isPriceKey = (key: string) => key.includes("cost") && !PRICES_SHOWN_ELSEWHERE.has(key); +const isUsdPriceKey = (key: string) => + (key.includes("cost_per_") || key.includes("token_cost")) && !key.includes("_dbu_"); + +const isPerTokenPrice = (key: string) => /_token(_|$)/.test(key); + +const isPriceKey = (key: string) => isUsdPriceKey(key) && !PRICES_SHOWN_ELSEWHERE.has(key); const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value); const isExtraPrice = (entry: [string, unknown]): entry is [string, number] => isPriceKey(entry[0]) && isFinitePrice(entry[1]); +const formatPrice = (key: string, price: number) => + isPerTokenPrice(key) + ? `$${priceFormat.format(price * TOKENS_PER_LISTED_PRICE)} per 1M tokens` + : `$${priceFormat.format(price)}`; + export function ModelExtraPrices({ model }: { model: Record }) { const prices = Object.entries(model) .filter(isExtraPrice) @@ -23,7 +35,7 @@ export function ModelExtraPrices({ model }: { model: Record }) {prices.map(([key, price]) => (
{key}
-
${priceFormat.format(price)}
+
{formatPrice(key, price)}
))}