From ea4e75378ce065fca5c8bd90191da7e0741d4c82 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:09:24 -0700 Subject: [PATCH] fix(router): list every finite cost map price on /model_group/info and the model hub Read prices from the deployment's cost map entry as well as its model info, so prices the typed model info leaves out (per-resolution image prices, DBU prices) reach the group. Drop infinite and NaN prices instead of failing the response, and show the extra prices in the dashboard's model hub detail panels. --- litellm/cost_calculator.py | 8 +-- litellm/router.py | 17 +++--- litellm/types/router.py | 32 +++++------ .../proxy_server/test_routes_model_info.py | 20 +++++-- tests/unit/test_router/test_router.py | 53 +++++++++++++++++-- .../AIHub/ModelExtraPrices.test.tsx | 39 ++++++++++++++ .../src/components/AIHub/ModelExtraPrices.tsx | 32 +++++++++++ .../src/components/AIHub/ModelHubTable.tsx | 2 + .../src/components/public_model_hub.tsx | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 10 files changed, 173 insertions(+), 34 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx create mode 100644 ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 86b22069a39..9def708e691 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -909,7 +909,7 @@ def _bedrock_pricing_info( def cost_map_key_prices_anything(key: str) -> bool: - entry: Final = _raw_cost_map_entry(key) + entry: Final = raw_cost_map_entry(key) return entry is not None and _cost_map_entry_prices_anything(entry) @@ -2358,7 +2358,7 @@ def _deployment_model_info( ) -> ModelInfo | None: if not custom_pricing: return None - registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None + registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None if registered_deployment_info is not None: return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id if litellm_logging_obj is None: @@ -2431,7 +2431,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, . ) -def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: +def raw_cost_map_entry(key: str) -> Mapping[str, object] | None: raw_entry: Final = litellm.model_cost.get(key) return raw_entry if isinstance(raw_entry, Mapping) else None @@ -2475,7 +2475,7 @@ def pricing_entry_for_cost_calc( ) if resolved is None: return None - return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved def ocr_cost( diff --git a/litellm/router.py b/litellm/router.py index 7a39a5c2079..9e8f958ede8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -80,6 +80,7 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) +from litellm.cost_calculator import raw_cost_map_entry from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes @@ -510,6 +511,12 @@ RouterStrategySelector: TypeAlias = ( ) +def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: + cost_map_key: Final = model_info.get("key") + entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None + return model_info_prices(entry) if entry is not None else MappingProxyType({}) + + def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: """Single owner of the environment-gating rule: a deployment whose model_info names `supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list. @@ -12585,6 +12592,7 @@ class Router: "model_group": user_facing_model_group_name, "providers": [llm_provider], "description": model_group_description, + **_cost_map_prices(model_info), **model_info, "supports_fast_mode": True, "supported_reasoning_efforts": None, @@ -12619,14 +12627,7 @@ class Router: ) ): model_group_info.max_output_tokens = model_info["max_output_tokens"] - group_prices = model_group_info.prices - model_group_info = model_group_info.model_copy( - update={ - key: price - for key, price in model_info_prices(model_info).items() - if key not in group_prices or price > group_prices[key] - } - ) + model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info}) if ( model_info.get("supports_parallel_function_calling", None) is not None and model_info["supports_parallel_function_calling"] is True diff --git a/litellm/types/router.py b/litellm/types/router.py index 12122087e29..48434abd35e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import math from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType @@ -70,9 +71,6 @@ from .utils import ( ModelResponse, StandardLoggingRoutingDecision, ) -from .utils import ( - ModelInfo as _CostMapModelInfo, -) from .utils import ( server_owned_wif_litellm_params as _server_owned_wif_litellm_params, ) @@ -976,11 +974,6 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: return get_type_hints(model_class) -_MODEL_INFO_PRICE_KEYS: Final[frozenset[str]] = frozenset( - key for key in _CostMapModelInfo.__required_keys__ | _CostMapModelInfo.__optional_keys__ if "cost" in key -) - - _RAW_FIELDS: Final = TypeAdapter(Mapping[str, object]) @@ -988,15 +981,14 @@ def price_as_float(value: object) -> float | None: if isinstance(value, bool) or not isinstance(value, int | float | str): return None try: - return float(value) + price: Final = float(value) except ValueError: return None + return price if math.isfinite(price) else None def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: - candidates: Final = ( - (key, price_as_float(value)) for key, value in model_info.items() if key in _MODEL_INFO_PRICE_KEYS - ) + candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key) return MappingProxyType({key: price for key, price in candidates if price is not None}) @@ -1006,7 +998,7 @@ class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth): class ModelGroupInfo(LiteLLMBaseModel): - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", json_schema_extra={"additionalProperties": True}) __pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict model_group: str @@ -1050,13 +1042,23 @@ class ModelGroupInfo(LiteLLMBaseModel): if not isinstance(data, Mapping): return data fields: Final = _RAW_FIELDS.validate_python(data) - declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields} - return {**model_info_prices(fields), **declared} + declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key} + return {**declared, **model_info_prices(fields)} @property def prices(self) -> Mapping[str, float]: return model_info_prices({**vars(self), **self.__pydantic_extra__}) + def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo": + current: Final = self.prices + return self.model_copy( + update={ + key: price + for key, price in model_info_prices(model_info).items() + if key not in current or price > current[key] + } + ) + class AssistantsTypedDict(TypedDict): custom_llm_provider: Literal["azure", "openai"] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 1a21370431f..a2ad764dfa6 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -973,7 +973,11 @@ def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyP @pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)]) def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment( - client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + team_id: str | None, + lists_batch: bool, ): from litellm.proxy._types import LitellmUserRoles @@ -994,7 +998,7 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price( - client, auth_as, monkeypatch + client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch ): from litellm.proxy._types import LitellmUserRoles @@ -1008,6 +1012,9 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep "output_cost_per_image": 0.5, "output_cost_per_image_token": 2e-05, "input_cost_per_token_priority": 6e-06, + "output_cost_per_image_4K": 0.16, + "output_cost_per_image_2K": "Infinity", + "input_cost_per_token": "NaN", "custom_tags": ["images"], }, }, @@ -1038,16 +1045,23 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep "output_cost_per_image_token", "input_cost_per_token_priority", "input_cost_per_token_above_272k_tokens", + "output_cost_per_image_4K", + "output_cost_per_image_2K", ) } == { "output_cost_per_image": 0.5, "output_cost_per_image_token": 4e-05, "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 8e-06, + "output_cost_per_image_4K": 0.16, + "output_cost_per_image_2K": None, } + assert group["input_cost_per_token"] is None -def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(client, auth_as, monkeypatch): +def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param( + client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch +): from litellm.proxy._types import LitellmUserRoles router: Final = litellm.Router( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 672ee1ccde9..a0f29dd3e59 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2495,7 +2495,7 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar @pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)]) -def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order): +def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order: tuple[int, int]): deployment_prices: Final = ( { "output_cost_per_image": 0.5, @@ -2509,6 +2509,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, + "input_dbu_cost_per_token": 7e-07, }, ) router: Final = litellm.Router( @@ -2520,6 +2521,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "id": f"tiered-deployment-{index}", **deployment_prices[index], "output_cost_per_video_per_second": "free", + "output_cost_per_second": "inf", "search_context_cost_per_query": {"search_context_size_low": 0.01}, }, } @@ -2537,12 +2539,54 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments( "input_cost_per_token_priority": 6e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_dbu_cost_per_token": 7e-07, } assert reported.keys().isdisjoint( - {"output_cost_per_video_per_second", "search_context_cost_per_query", "litellm_provider", "key", "id"} + { + "output_cost_per_video_per_second", + "output_cost_per_second", + "search_context_cost_per_query", + "litellm_provider", + "key", + "id", + } ) +def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(monkeypatch: pytest.MonkeyPatch): + from litellm.utils import _invalidate_model_cost_lowercase_map + + cost_map_prices: Final = { + "output_cost_per_image": 0.08, + "output_cost_per_image_0.5K": 0.06, + "output_cost_per_image_4K": 0.16, + "input_dbu_cost_per_token": 0.0002, + } + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + "made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices}, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "resolution-group", + "litellm_params": {"model": "openai/made-up-resolution-model", "api_key": "fake"}, + } + ] + ) + + info: Final = router.get_model_group_info(model_group="resolution-group") + + assert info is not None + reported: Final = info.model_dump() + assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices + + def test_model_group_info_still_validates_from_attributes(): source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06) @@ -2898,9 +2942,12 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto ("not-a-number", None), (True, None), ({"input_cost_per_token": 1e-05}, None), + ("Infinity", None), + ("-inf", None), + (float("nan"), None), ], ) -def test_price_as_float(value, expected): +def test_price_as_float(value: object, expected: float | None): from litellm.types.router import price_as_float assert price_as_float(value) == expected diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx new file mode 100644 index 00000000000..97934392993 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx @@ -0,0 +1,39 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { ModelExtraPrices } from "./ModelExtraPrices"; + +describe("ModelExtraPrices", () => { + it("lists every numeric price besides the per-token pair, sorted by name", () => { + render( + , + ); + const terms = screen.getAllByRole("term").map((term) => term.textContent); + const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent); + expect(terms.map((term, index) => [term, definitions[index]])).toEqual([ + ["input_cost_per_token_priority", "$0.0000025"], + ["output_cost_per_image", "$0.04"], + ["output_cost_per_image_4K", "$0.16"], + ]); + }); + + it("renders nothing when the model lists only the per-token prices", () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx new file mode 100644 index 00000000000..bb24dd5e115 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx @@ -0,0 +1,32 @@ +const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]); + +const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 }); + +const isPriceKey = (key: string) => key.includes("cost") && !PRICES_SHOWN_ELSEWHERE.has(key); + +const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value); + +const isExtraPrice = (entry: [string, unknown]): entry is [string, number] => + isPriceKey(entry[0]) && isFinitePrice(entry[1]); + +export function ModelExtraPrices({ model }: { model: Record }) { + const prices = Object.entries(model) + .filter(isExtraPrice) + .sort(([a], [b]) => a.localeCompare(b)); + if (prices.length === 0) { + return null; + } + return ( +
+

Other Prices (USD):

+
+ {prices.map(([key, price]) => ( +
+
{key}
+
${priceFormat.format(price)}
+
+ ))} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 59f26b72481..ab7156256c1 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -56,6 +56,7 @@ import { } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; import { ModelCustomMetadata } from "./ModelCustomMetadata"; +import { ModelExtraPrices } from "./ModelExtraPrices"; interface ModelHubTableProps { accessToken: string | null; @@ -788,6 +789,7 @@ const ModelHubTable: React.FC = ({ : "Not specified"}

+ diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 59da41e7c05..1aaf7760ef2 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -55,6 +55,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; +import { ModelExtraPrices } from "@/components/AIHub/ModelExtraPrices"; import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders"; interface PublicModelHubProps { @@ -884,6 +885,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded : "Not specified"}

+ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 60ef2b0ed76..5f528035fd7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38521,7 +38521,7 @@ export interface components { /** Tpm */ tpm?: number | null; } & { - [key: string]: number; + [key: string]: unknown; }; /** ModelInfoDelete */ ModelInfoDelete: {