diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 86b22069a39..9def708e691 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -909,7 +909,7 @@ def _bedrock_pricing_info(
def cost_map_key_prices_anything(key: str) -> bool:
- entry: Final = _raw_cost_map_entry(key)
+ entry: Final = raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
@@ -2358,7 +2358,7 @@ def _deployment_model_info(
) -> ModelInfo | None:
if not custom_pricing:
return None
- registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None
+ registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None
if registered_deployment_info is not None:
return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id
if litellm_logging_obj is None:
@@ -2431,7 +2431,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
)
-def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
+def raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@@ -2475,7 +2475,7 @@ def pricing_entry_for_cost_calc(
)
if resolved is None:
return None
- return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved
+ return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved
def ocr_cost(
diff --git a/litellm/router.py b/litellm/router.py
index 7a39a5c2079..9e8f958ede8 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -80,6 +80,7 @@ from litellm.constants import (
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
+from litellm.cost_calculator import raw_cost_map_entry
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
@@ -510,6 +511,12 @@ RouterStrategySelector: TypeAlias = (
)
+def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
+ cost_map_key: Final = model_info.get("key")
+ entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
+ return model_info_prices(entry) if entry is not None else MappingProxyType({})
+
+
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
"""Single owner of the environment-gating rule: a deployment whose model_info names
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
@@ -12585,6 +12592,7 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
+ **_cost_map_prices(model_info),
**model_info,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
@@ -12619,14 +12627,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
- group_prices = model_group_info.prices
- model_group_info = model_group_info.model_copy(
- update={
- key: price
- for key, price in model_info_prices(model_info).items()
- if key not in group_prices or price > group_prices[key]
- }
- )
+ model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info})
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 12122087e29..48434abd35e 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
+import math
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
@@ -70,9 +71,6 @@ from .utils import (
ModelResponse,
StandardLoggingRoutingDecision,
)
-from .utils import (
- ModelInfo as _CostMapModelInfo,
-)
from .utils import (
server_owned_wif_litellm_params as _server_owned_wif_litellm_params,
)
@@ -976,11 +974,6 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
return get_type_hints(model_class)
-_MODEL_INFO_PRICE_KEYS: Final[frozenset[str]] = frozenset(
- key for key in _CostMapModelInfo.__required_keys__ | _CostMapModelInfo.__optional_keys__ if "cost" in key
-)
-
-
_RAW_FIELDS: Final = TypeAdapter(Mapping[str, object])
@@ -988,15 +981,14 @@ def price_as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float | str):
return None
try:
- return float(value)
+ price: Final = float(value)
except ValueError:
return None
+ return price if math.isfinite(price) else None
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
- candidates: Final = (
- (key, price_as_float(value)) for key, value in model_info.items() if key in _MODEL_INFO_PRICE_KEYS
- )
+ candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key)
return MappingProxyType({key: price for key, price in candidates if price is not None})
@@ -1006,7 +998,7 @@ class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth):
class ModelGroupInfo(LiteLLMBaseModel):
- model_config = ConfigDict(extra="allow")
+ model_config = ConfigDict(extra="allow", json_schema_extra={"additionalProperties": True})
__pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict
model_group: str
@@ -1050,13 +1042,23 @@ class ModelGroupInfo(LiteLLMBaseModel):
if not isinstance(data, Mapping):
return data
fields: Final = _RAW_FIELDS.validate_python(data)
- declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields}
- return {**model_info_prices(fields), **declared}
+ declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key}
+ return {**declared, **model_info_prices(fields)}
@property
def prices(self) -> Mapping[str, float]:
return model_info_prices({**vars(self), **self.__pydantic_extra__})
+ def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo":
+ current: Final = self.prices
+ return self.model_copy(
+ update={
+ key: price
+ for key, price in model_info_prices(model_info).items()
+ if key not in current or price > current[key]
+ }
+ )
+
class AssistantsTypedDict(TypedDict):
custom_llm_provider: Literal["azure", "openai"]
diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py
index 1a21370431f..a2ad764dfa6 100644
--- a/tests/unit/proxy/proxy_server/test_routes_model_info.py
+++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py
@@ -973,7 +973,11 @@ def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyP
@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
- client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool
+ client: TestClient,
+ auth_as: Callable[..., AbstractContextManager[object]],
+ monkeypatch: pytest.MonkeyPatch,
+ team_id: str | None,
+ lists_batch: bool,
):
from litellm.proxy._types import LitellmUserRoles
@@ -994,7 +998,7 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price(
- client, auth_as, monkeypatch
+ client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy._types import LitellmUserRoles
@@ -1008,6 +1012,9 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 2e-05,
"input_cost_per_token_priority": 6e-06,
+ "output_cost_per_image_4K": 0.16,
+ "output_cost_per_image_2K": "Infinity",
+ "input_cost_per_token": "NaN",
"custom_tags": ["images"],
},
},
@@ -1038,16 +1045,23 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_token",
"input_cost_per_token_priority",
"input_cost_per_token_above_272k_tokens",
+ "output_cost_per_image_4K",
+ "output_cost_per_image_2K",
)
} == {
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
+ "output_cost_per_image_4K": 0.16,
+ "output_cost_per_image_2K": None,
}
+ assert group["input_cost_per_token"] is None
-def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(client, auth_as, monkeypatch):
+def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(
+ client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
+):
from litellm.proxy._types import LitellmUserRoles
router: Final = litellm.Router(
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index 672ee1ccde9..a0f29dd3e59 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2495,7 +2495,7 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar
@pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)])
-def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order):
+def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order: tuple[int, int]):
deployment_prices: Final = (
{
"output_cost_per_image": 0.5,
@@ -2509,6 +2509,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
+ "input_dbu_cost_per_token": 7e-07,
},
)
router: Final = litellm.Router(
@@ -2520,6 +2521,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"id": f"tiered-deployment-{index}",
**deployment_prices[index],
"output_cost_per_video_per_second": "free",
+ "output_cost_per_second": "inf",
"search_context_cost_per_query": {"search_context_size_low": 0.01},
},
}
@@ -2537,12 +2539,54 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
+ "input_dbu_cost_per_token": 7e-07,
}
assert reported.keys().isdisjoint(
- {"output_cost_per_video_per_second", "search_context_cost_per_query", "litellm_provider", "key", "id"}
+ {
+ "output_cost_per_video_per_second",
+ "output_cost_per_second",
+ "search_context_cost_per_query",
+ "litellm_provider",
+ "key",
+ "id",
+ }
)
+def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(monkeypatch: pytest.MonkeyPatch):
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ cost_map_prices: Final = {
+ "output_cost_per_image": 0.08,
+ "output_cost_per_image_0.5K": 0.06,
+ "output_cost_per_image_4K": 0.16,
+ "input_dbu_cost_per_token": 0.0002,
+ }
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ **litellm.model_cost,
+ "made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices},
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "resolution-group",
+ "litellm_params": {"model": "openai/made-up-resolution-model", "api_key": "fake"},
+ }
+ ]
+ )
+
+ info: Final = router.get_model_group_info(model_group="resolution-group")
+
+ assert info is not None
+ reported: Final = info.model_dump()
+ assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices
+
+
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
@@ -2898,9 +2942,12 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
("not-a-number", None),
(True, None),
({"input_cost_per_token": 1e-05}, None),
+ ("Infinity", None),
+ ("-inf", None),
+ (float("nan"), None),
],
)
-def test_price_as_float(value, expected):
+def test_price_as_float(value: object, expected: float | None):
from litellm.types.router import price_as_float
assert price_as_float(value) == expected
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
new file mode 100644
index 00000000000..97934392993
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
@@ -0,0 +1,39 @@
+import { render, screen } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+import { ModelExtraPrices } from "./ModelExtraPrices";
+
+describe("ModelExtraPrices", () => {
+ it("lists every numeric price besides the per-token pair, sorted by name", () => {
+ render(
+ Other Prices (USD):
+ {prices.map(([key, price]) => (
+
+