From 286292b6cb6b7c5184b275a2db73e3a3342188e8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 28 Oct 2026 02:41:46 -0700
Subject: [PATCH 1/8] fix(router): carry every cost map price on
/model_group/info
The model group info endpoints only showed base input and output token
prices. A group now reports every scalar price from its deployments'
model info, at the highest value across the group, under the same names
/model/info uses. Non-numeric values and non-price keys are still dropped
---
litellm/proxy/auth/auth_utils.py | 6 +-
litellm/router.py | 35 ++----
litellm/types/router.py | 63 +++++++++-
.../proxy_server/test_routes_model_info.py | 111 ++++++++++++++++--
tests/unit/test_router/test_router.py | 65 +++++++++-
ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 ++--
6 files changed, 248 insertions(+), 52 deletions(-)
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index aa6ddae438d..6d7098d6b5a 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -2,7 +2,7 @@ import importlib.util
import os
import re
import sys
-from collections.abc import Collection, Iterator, Mapping
+from collections.abc import Collection, Iterator, Mapping, Sequence
from functools import lru_cache
from itertools import accumulate, chain, repeat
from logging import Logger
@@ -38,7 +38,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
)
from litellm.types.router import (
- CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
+ ConfigurableClientsideParamsCustomAuth,
Deployment,
reject_server_owned_wif_fields,
server_owned_wif_fields_present,
@@ -189,7 +189,7 @@ def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool:
def _is_param_allowed(
param: str,
request_body_value: object,
- configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
+ configurable_clientside_auth_params: Sequence[str | ConfigurableClientsideParamsCustomAuth] | None,
) -> bool:
"""
Check if param is a str or dict and if request_body_value is in the list of allowed values
diff --git a/litellm/router.py b/litellm/router.py
index c8153b596d6..7a39a5c2079 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -415,6 +415,8 @@ from litellm.types.router import (
SearchToolTypedDict,
TaggedPreRoutingStrategy,
holds_secret_pointer,
+ model_info_prices,
+ price_as_float,
)
from litellm.types.services import ServiceTypes
from litellm.types.utils import (
@@ -508,15 +510,6 @@ RouterStrategySelector: TypeAlias = (
)
-def _cost_value_as_float(value: str | float | None) -> float | None:
- if value is None:
- return None
- try:
- return float(value)
- except (TypeError, ValueError):
- return None
-
-
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
"""Single owner of the environment-gating rule: a deployment whose model_info names
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
@@ -12570,8 +12563,8 @@ class Router:
# Get mode from database model_info if available, otherwise default to "chat"
db_model_info = model.get("model_info", {})
mode = db_model_info.get("mode", "chat")
- input_cost_per_token = _cost_value_as_float(db_model_info.get("input_cost_per_token"))
- output_cost_per_token = _cost_value_as_float(db_model_info.get("output_cost_per_token"))
+ input_cost_per_token = price_as_float(db_model_info.get("input_cost_per_token"))
+ output_cost_per_token = price_as_float(db_model_info.get("output_cost_per_token"))
model_info = ModelMapInfo(
key=model_group,
@@ -12626,18 +12619,14 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
- _input_cost_per_token = _cost_value_as_float(model_info.get("input_cost_per_token"))
- if _input_cost_per_token is not None and (
- model_group_info.input_cost_per_token is None
- or _input_cost_per_token > (model_group_info.input_cost_per_token or 0.0)
- ):
- model_group_info.input_cost_per_token = _input_cost_per_token
- _output_cost_per_token = _cost_value_as_float(model_info.get("output_cost_per_token"))
- if _output_cost_per_token is not None and (
- model_group_info.output_cost_per_token is None
- or _output_cost_per_token > (model_group_info.output_cost_per_token or 0.0)
- ):
- model_group_info.output_cost_per_token = _output_cost_per_token
+ group_prices = model_group_info.prices
+ model_group_info = model_group_info.model_copy(
+ update={
+ key: price
+ for key, price in model_info_prices(model_info).items()
+ if key not in group_prices or price > group_prices[key]
+ }
+ )
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True
diff --git a/litellm/types/router.py b/litellm/types/router.py
index ba9b82e2a57..12122087e29 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -6,6 +6,7 @@ import datetime
import enum
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
+from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
@@ -22,7 +23,17 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import httpx
from openai import Timeout as SDKTimeout
-from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
+from pydantic import (
+ BeforeValidator,
+ ConfigDict,
+ Field,
+ JsonValue,
+ StrictInt,
+ TypeAdapter,
+ field_validator,
+ model_validator,
+ with_config,
+)
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from litellm._logging import verbose_logger
@@ -59,6 +70,9 @@ from .utils import (
ModelResponse,
StandardLoggingRoutingDecision,
)
+from .utils import (
+ ModelInfo as _CostMapModelInfo,
+)
from .utils import (
server_owned_wif_litellm_params as _server_owned_wif_litellm_params,
)
@@ -962,7 +976,39 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
return get_type_hints(model_class)
+_MODEL_INFO_PRICE_KEYS: Final[frozenset[str]] = frozenset(
+ key for key in _CostMapModelInfo.__required_keys__ | _CostMapModelInfo.__optional_keys__ if "cost" in key
+)
+
+
+_RAW_FIELDS: Final = TypeAdapter(Mapping[str, object])
+
+
+def price_as_float(value: object) -> float | None:
+ if isinstance(value, bool) or not isinstance(value, int | float | str):
+ return None
+ try:
+ return float(value)
+ except ValueError:
+ return None
+
+
+def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
+ candidates: Final = (
+ (key, price_as_float(value)) for key, value in model_info.items() if key in _MODEL_INFO_PRICE_KEYS
+ )
+ return MappingProxyType({key: price for key, price in candidates if price is not None})
+
+
+@with_config(ConfigDict(extra="ignore"))
+class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth):
+ pass
+
+
class ModelGroupInfo(LiteLLMBaseModel):
+ model_config = ConfigDict(extra="allow")
+ __pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict
+
model_group: str
providers: list[str]
max_input_tokens: float | None = None
@@ -988,7 +1034,7 @@ class ModelGroupInfo(LiteLLMBaseModel):
supports_fast_mode: bool = Field(default=False)
supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None)
supported_openai_params: list[str] | None = Field(default=[])
- configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
+ configurable_clientside_auth_params: Sequence[str | ModelGroupClientsideAuthParam] | None = None
description: str | None = None
supported_endpoints: tuple[str, ...] | None = None
@@ -998,6 +1044,19 @@ class ModelGroupInfo(LiteLLMBaseModel):
data[field_name] = False
super().__init__(**data)
+ @model_validator(mode="before")
+ @classmethod
+ def _keep_only_price_extras(cls, data: object) -> object:
+ if not isinstance(data, Mapping):
+ return data
+ fields: Final = _RAW_FIELDS.validate_python(data)
+ declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields}
+ return {**model_info_prices(fields), **declared}
+
+ @property
+ def prices(self) -> Mapping[str, float]:
+ return model_info_prices({**vars(self), **self.__pydantic_extra__})
+
class AssistantsTypedDict(TypedDict):
custom_llm_provider: Literal["azure", "openai"]
diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py
index 9993361d930..1a21370431f 100644
--- a/tests/unit/proxy/proxy_server/test_routes_model_info.py
+++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py
@@ -955,20 +955,9 @@ def _azure_group_deployment(sku_host: str, mode: str, **model_info: object) -> d
}
-@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
-def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
- client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool
-):
- from litellm.proxy._types import LitellmUserRoles
+def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.agent_endpoints import model_list_helpers
- router: Final = litellm.Router(
- model_list=[
- _azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]),
- _azure_group_deployment("batch-sku", "batch", team_id="team-a"),
- ]
- )
-
async def append_agents_to_model_group(*, model_groups, user_api_key_dict):
return model_groups
@@ -981,6 +970,21 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group)
)
+
+@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
+def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
+ client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool
+):
+ from litellm.proxy._types import LitellmUserRoles
+
+ router: Final = litellm.Router(
+ model_list=[
+ _azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]),
+ _azure_group_deployment("batch-sku", "batch", team_id="team-a"),
+ ]
+ )
+ _serve_model_groups_from(router, monkeypatch)
+
with auth_as(LitellmUserRoles.PROXY_ADMIN, team_id=team_id):
response: Final = client.get("/model_group/info")
@@ -989,6 +993,89 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch
+def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price(
+ client, auth_as, monkeypatch
+):
+ from litellm.proxy._types import LitellmUserRoles
+
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "image-pricing-group",
+ "litellm_params": {"model": "openai/made-up-image-model-a", "api_key": "dummy-key"},
+ "model_info": {
+ "id": "image-deployment-a",
+ "output_cost_per_image": 0.5,
+ "output_cost_per_image_token": 2e-05,
+ "input_cost_per_token_priority": 6e-06,
+ "custom_tags": ["images"],
+ },
+ },
+ {
+ "model_name": "image-pricing-group",
+ "litellm_params": {"model": "openai/made-up-image-model-b", "api_key": "dummy-key"},
+ "model_info": {
+ "id": "image-deployment-b",
+ "output_cost_per_image": 0.25,
+ "output_cost_per_image_token": 4e-05,
+ "input_cost_per_token_above_272k_tokens": 8e-06,
+ },
+ },
+ ]
+ )
+ _serve_model_groups_from(router, monkeypatch)
+
+ with auth_as(LitellmUserRoles.PROXY_ADMIN):
+ response: Final = client.get("/model_group/info")
+
+ assert response.status_code == 200, response.text
+ (group,) = response.json()["data"]
+ assert group["custom_tags"] == ["images"]
+ assert {
+ key: group.get(key)
+ for key in (
+ "output_cost_per_image",
+ "output_cost_per_image_token",
+ "input_cost_per_token_priority",
+ "input_cost_per_token_above_272k_tokens",
+ )
+ } == {
+ "output_cost_per_image": 0.5,
+ "output_cost_per_image_token": 4e-05,
+ "input_cost_per_token_priority": 6e-06,
+ "input_cost_per_token_above_272k_tokens": 8e-06,
+ }
+
+
+def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(client, auth_as, monkeypatch):
+ from litellm.proxy._types import LitellmUserRoles
+
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "clientside-auth-group",
+ "litellm_params": {
+ "model": "openai/made-up-clientside-auth-model",
+ "api_key": "dummy-key",
+ "configurable_clientside_auth_params": [
+ "api_key",
+ {"api_base": "^https://.*\\.example\\.com$", "unlisted_setting": "admin-only"},
+ ],
+ },
+ "model_info": {"id": "clientside-auth-deployment"},
+ }
+ ]
+ )
+ _serve_model_groups_from(router, monkeypatch)
+
+ with auth_as(LitellmUserRoles.PROXY_ADMIN):
+ response: Final = client.get("/model_group/info")
+
+ assert response.status_code == 200, response.text
+ (group,) = response.json()["data"]
+ assert group["configurable_clientside_auth_params"] == ["api_key", {"api_base": "^https://.*\\.example\\.com$"}]
+
+
# ---------------------------------------------------------------------------
# GET /v2/model/info?exclude_auto_routers
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index d9b655a0aff..672ee1ccde9 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2494,6 +2494,63 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar
assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0
+@pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)])
+def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order):
+ deployment_prices: Final = (
+ {
+ "output_cost_per_image": 0.5,
+ "output_cost_per_image_token": 2e-05,
+ "input_cost_per_token_priority": 3e-06,
+ "input_cost_per_token_above_272k_tokens": 8e-06,
+ },
+ {
+ "output_cost_per_image": 0.25,
+ "output_cost_per_image_token": 4e-05,
+ "input_cost_per_token_priority": 6e-06,
+ "input_cost_per_token_above_272k_tokens": 4e-06,
+ "input_cost_per_token_batches": 1e-06,
+ },
+ )
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "tiered-pricing-group",
+ "litellm_params": {"model": f"openai/made-up-tiered-model-{index}", "api_key": "fake"},
+ "model_info": {
+ "id": f"tiered-deployment-{index}",
+ **deployment_prices[index],
+ "output_cost_per_video_per_second": "free",
+ "search_context_cost_per_query": {"search_context_size_low": 0.01},
+ },
+ }
+ for index in deployment_order
+ ]
+ )
+
+ info: Final = router.get_model_group_info(model_group="tiered-pricing-group")
+
+ assert info is not None
+ reported: Final = info.model_dump()
+ assert {key: reported.get(key) for key in deployment_prices[1]} == {
+ "output_cost_per_image": 0.5,
+ "output_cost_per_image_token": 4e-05,
+ "input_cost_per_token_priority": 6e-06,
+ "input_cost_per_token_above_272k_tokens": 8e-06,
+ "input_cost_per_token_batches": 1e-06,
+ }
+ assert reported.keys().isdisjoint(
+ {"output_cost_per_video_per_second", "search_context_cost_per_query", "litellm_provider", "key", "id"}
+ )
+
+
+def test_model_group_info_still_validates_from_attributes():
+ source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
+
+ info: Final = ModelGroupInfo.model_validate(source, from_attributes=True)
+
+ assert (info.model_group, info.providers, info.input_cost_per_token) == ("attribute-group", ["openai"], 1e-06)
+
+
def test_model_group_info_description_from_model_info():
"""
model_info.description set in the config should surface on ModelGroupInfo,
@@ -2839,12 +2896,14 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
(5, 5.0),
(None, None),
("not-a-number", None),
+ (True, None),
+ ({"input_cost_per_token": 1e-05}, None),
],
)
-def test_cost_value_as_float(value, expected):
- from litellm.router import _cost_value_as_float
+def test_price_as_float(value, expected):
+ from litellm.types.router import price_as_float
- assert _cost_value_as_float(value) == expected
+ assert price_as_float(value) == expected
def test_model_group_info_with_stringified_cost_values():
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 8af76e4d16f..60ef2b0ed76 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -30517,17 +30517,12 @@ export interface components {
worker_registry?: components["schemas"]["WorkerRegistryEntry"][] | null;
};
/** ConfigurableClientsideParamsCustomAuth */
- "ConfigurableClientsideParamsCustomAuth-Input": {
+ ConfigurableClientsideParamsCustomAuth: {
/** Api Base */
api_base: string;
} & {
[key: string]: unknown;
};
- /** ConfigurableClientsideParamsCustomAuth */
- "ConfigurableClientsideParamsCustomAuth-Output": {
- /** Api Base */
- api_base: string;
- };
/** ContainerAuto */
ContainerAuto: {
/** File Ids */
@@ -35354,7 +35349,7 @@ export interface components {
/** Complexity Router Default Model */
complexity_router_default_model?: string | null;
/** Configurable Clientside Auth Params */
- configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Input"])[] | null;
+ configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth"])[] | null;
/** Cost Per Second */
cost_per_second?: number | null;
/** Custom Llm Provider */
@@ -38426,10 +38421,15 @@ export interface components {
*/
warn_within_days: number;
};
+ /** ModelGroupClientsideAuthParam */
+ ModelGroupClientsideAuthParam: {
+ /** Api Base */
+ api_base: string;
+ };
/** ModelGroupInfoProxy */
ModelGroupInfoProxy: {
/** Configurable Clientside Auth Params */
- configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null;
+ configurable_clientside_auth_params?: (string | components["schemas"]["ModelGroupClientsideAuthParam"])[] | null;
/** Custom Metadata */
custom_metadata?: {
[key: string]: string | number | boolean;
@@ -38520,6 +38520,8 @@ export interface components {
supports_web_search: boolean;
/** Tpm */
tpm?: number | null;
+ } & {
+ [key: string]: number;
};
/** ModelInfoDelete */
ModelInfoDelete: {
@@ -50740,7 +50742,7 @@ export interface components {
/** Complexity Router Default Model */
complexity_router_default_model?: string | null;
/** Configurable Clientside Auth Params */
- configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Input"])[] | null;
+ configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth"])[] | null;
/** Cost Per Second */
cost_per_second?: number | null;
/** Custom Llm Provider */
From ea4e75378ce065fca5c8bd90191da7e0741d4c82 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 28 Oct 2026 05:09:24 -0700
Subject: [PATCH 2/8] fix(router): list every finite cost map price on
/model_group/info and the model hub
Read prices from the deployment's cost map entry as well as its model info, so prices the typed model info leaves out (per-resolution image prices, DBU prices) reach the group. Drop infinite and NaN prices instead of failing the response, and show the extra prices in the dashboard's model hub detail panels.
---
litellm/cost_calculator.py | 8 +--
litellm/router.py | 17 +++---
litellm/types/router.py | 32 +++++------
.../proxy_server/test_routes_model_info.py | 20 +++++--
tests/unit/test_router/test_router.py | 53 +++++++++++++++++--
.../AIHub/ModelExtraPrices.test.tsx | 39 ++++++++++++++
.../src/components/AIHub/ModelExtraPrices.tsx | 32 +++++++++++
.../src/components/AIHub/ModelHubTable.tsx | 2 +
.../src/components/public_model_hub.tsx | 2 +
ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +-
10 files changed, 173 insertions(+), 34 deletions(-)
create mode 100644 ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 86b22069a39..9def708e691 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -909,7 +909,7 @@ def _bedrock_pricing_info(
def cost_map_key_prices_anything(key: str) -> bool:
- entry: Final = _raw_cost_map_entry(key)
+ entry: Final = raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
@@ -2358,7 +2358,7 @@ def _deployment_model_info(
) -> ModelInfo | None:
if not custom_pricing:
return None
- registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None
+ registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None
if registered_deployment_info is not None:
return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id
if litellm_logging_obj is None:
@@ -2431,7 +2431,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
)
-def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
+def raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@@ -2475,7 +2475,7 @@ def pricing_entry_for_cost_calc(
)
if resolved is None:
return None
- return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved
+ return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved
def ocr_cost(
diff --git a/litellm/router.py b/litellm/router.py
index 7a39a5c2079..9e8f958ede8 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -80,6 +80,7 @@ from litellm.constants import (
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
+from litellm.cost_calculator import raw_cost_map_entry
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
@@ -510,6 +511,12 @@ RouterStrategySelector: TypeAlias = (
)
+def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
+ cost_map_key: Final = model_info.get("key")
+ entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
+ return model_info_prices(entry) if entry is not None else MappingProxyType({})
+
+
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
"""Single owner of the environment-gating rule: a deployment whose model_info names
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
@@ -12585,6 +12592,7 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
+ **_cost_map_prices(model_info),
**model_info,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
@@ -12619,14 +12627,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
- group_prices = model_group_info.prices
- model_group_info = model_group_info.model_copy(
- update={
- key: price
- for key, price in model_info_prices(model_info).items()
- if key not in group_prices or price > group_prices[key]
- }
- )
+ model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info})
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 12122087e29..48434abd35e 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
+import math
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
@@ -70,9 +71,6 @@ from .utils import (
ModelResponse,
StandardLoggingRoutingDecision,
)
-from .utils import (
- ModelInfo as _CostMapModelInfo,
-)
from .utils import (
server_owned_wif_litellm_params as _server_owned_wif_litellm_params,
)
@@ -976,11 +974,6 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
return get_type_hints(model_class)
-_MODEL_INFO_PRICE_KEYS: Final[frozenset[str]] = frozenset(
- key for key in _CostMapModelInfo.__required_keys__ | _CostMapModelInfo.__optional_keys__ if "cost" in key
-)
-
-
_RAW_FIELDS: Final = TypeAdapter(Mapping[str, object])
@@ -988,15 +981,14 @@ def price_as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float | str):
return None
try:
- return float(value)
+ price: Final = float(value)
except ValueError:
return None
+ return price if math.isfinite(price) else None
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
- candidates: Final = (
- (key, price_as_float(value)) for key, value in model_info.items() if key in _MODEL_INFO_PRICE_KEYS
- )
+ candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key)
return MappingProxyType({key: price for key, price in candidates if price is not None})
@@ -1006,7 +998,7 @@ class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth):
class ModelGroupInfo(LiteLLMBaseModel):
- model_config = ConfigDict(extra="allow")
+ model_config = ConfigDict(extra="allow", json_schema_extra={"additionalProperties": True})
__pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict
model_group: str
@@ -1050,13 +1042,23 @@ class ModelGroupInfo(LiteLLMBaseModel):
if not isinstance(data, Mapping):
return data
fields: Final = _RAW_FIELDS.validate_python(data)
- declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields}
- return {**model_info_prices(fields), **declared}
+ declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key}
+ return {**declared, **model_info_prices(fields)}
@property
def prices(self) -> Mapping[str, float]:
return model_info_prices({**vars(self), **self.__pydantic_extra__})
+ def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo":
+ current: Final = self.prices
+ return self.model_copy(
+ update={
+ key: price
+ for key, price in model_info_prices(model_info).items()
+ if key not in current or price > current[key]
+ }
+ )
+
class AssistantsTypedDict(TypedDict):
custom_llm_provider: Literal["azure", "openai"]
diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py
index 1a21370431f..a2ad764dfa6 100644
--- a/tests/unit/proxy/proxy_server/test_routes_model_info.py
+++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py
@@ -973,7 +973,11 @@ def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyP
@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
- client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool
+ client: TestClient,
+ auth_as: Callable[..., AbstractContextManager[object]],
+ monkeypatch: pytest.MonkeyPatch,
+ team_id: str | None,
+ lists_batch: bool,
):
from litellm.proxy._types import LitellmUserRoles
@@ -994,7 +998,7 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price(
- client, auth_as, monkeypatch
+ client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy._types import LitellmUserRoles
@@ -1008,6 +1012,9 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 2e-05,
"input_cost_per_token_priority": 6e-06,
+ "output_cost_per_image_4K": 0.16,
+ "output_cost_per_image_2K": "Infinity",
+ "input_cost_per_token": "NaN",
"custom_tags": ["images"],
},
},
@@ -1038,16 +1045,23 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_token",
"input_cost_per_token_priority",
"input_cost_per_token_above_272k_tokens",
+ "output_cost_per_image_4K",
+ "output_cost_per_image_2K",
)
} == {
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
+ "output_cost_per_image_4K": 0.16,
+ "output_cost_per_image_2K": None,
}
+ assert group["input_cost_per_token"] is None
-def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(client, auth_as, monkeypatch):
+def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(
+ client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
+):
from litellm.proxy._types import LitellmUserRoles
router: Final = litellm.Router(
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index 672ee1ccde9..a0f29dd3e59 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2495,7 +2495,7 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar
@pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)])
-def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order):
+def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order: tuple[int, int]):
deployment_prices: Final = (
{
"output_cost_per_image": 0.5,
@@ -2509,6 +2509,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
+ "input_dbu_cost_per_token": 7e-07,
},
)
router: Final = litellm.Router(
@@ -2520,6 +2521,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"id": f"tiered-deployment-{index}",
**deployment_prices[index],
"output_cost_per_video_per_second": "free",
+ "output_cost_per_second": "inf",
"search_context_cost_per_query": {"search_context_size_low": 0.01},
},
}
@@ -2537,12 +2539,54 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
+ "input_dbu_cost_per_token": 7e-07,
}
assert reported.keys().isdisjoint(
- {"output_cost_per_video_per_second", "search_context_cost_per_query", "litellm_provider", "key", "id"}
+ {
+ "output_cost_per_video_per_second",
+ "output_cost_per_second",
+ "search_context_cost_per_query",
+ "litellm_provider",
+ "key",
+ "id",
+ }
)
+def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(monkeypatch: pytest.MonkeyPatch):
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ cost_map_prices: Final = {
+ "output_cost_per_image": 0.08,
+ "output_cost_per_image_0.5K": 0.06,
+ "output_cost_per_image_4K": 0.16,
+ "input_dbu_cost_per_token": 0.0002,
+ }
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ **litellm.model_cost,
+ "made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices},
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "resolution-group",
+ "litellm_params": {"model": "openai/made-up-resolution-model", "api_key": "fake"},
+ }
+ ]
+ )
+
+ info: Final = router.get_model_group_info(model_group="resolution-group")
+
+ assert info is not None
+ reported: Final = info.model_dump()
+ assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices
+
+
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
@@ -2898,9 +2942,12 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
("not-a-number", None),
(True, None),
({"input_cost_per_token": 1e-05}, None),
+ ("Infinity", None),
+ ("-inf", None),
+ (float("nan"), None),
],
)
-def test_price_as_float(value, expected):
+def test_price_as_float(value: object, expected: float | None):
from litellm.types.router import price_as_float
assert price_as_float(value) == expected
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
new file mode 100644
index 00000000000..97934392993
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
@@ -0,0 +1,39 @@
+import { render, screen } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+import { ModelExtraPrices } from "./ModelExtraPrices";
+
+describe("ModelExtraPrices", () => {
+ it("lists every numeric price besides the per-token pair, sorted by name", () => {
+ render(
+ ,
+ );
+ const terms = screen.getAllByRole("term").map((term) => term.textContent);
+ const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent);
+ expect(terms.map((term, index) => [term, definitions[index]])).toEqual([
+ ["input_cost_per_token_priority", "$0.0000025"],
+ ["output_cost_per_image", "$0.04"],
+ ["output_cost_per_image_4K", "$0.16"],
+ ]);
+ });
+
+ it("renders nothing when the model lists only the per-token prices", () => {
+ const { container } = render(
+ ,
+ );
+ expect(container).toBeEmptyDOMElement();
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
new file mode 100644
index 00000000000..bb24dd5e115
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
@@ -0,0 +1,32 @@
+const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]);
+
+const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 });
+
+const isPriceKey = (key: string) => key.includes("cost") && !PRICES_SHOWN_ELSEWHERE.has(key);
+
+const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+
+const isExtraPrice = (entry: [string, unknown]): entry is [string, number] =>
+ isPriceKey(entry[0]) && isFinitePrice(entry[1]);
+
+export function ModelExtraPrices({ model }: { model: Record }) {
+ const prices = Object.entries(model)
+ .filter(isExtraPrice)
+ .sort(([a], [b]) => a.localeCompare(b));
+ if (prices.length === 0) {
+ return null;
+ }
+ return (
+
+
Other Prices (USD):
+
+ {prices.map(([key, price]) => (
+
+
- {key}
+ - ${priceFormat.format(price)}
+
+ ))}
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
index 59f26b72481..ab7156256c1 100644
--- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
+++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
@@ -56,6 +56,7 @@ import {
} from "@/lib/decisionModels";
import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden";
import { ModelCustomMetadata } from "./ModelCustomMetadata";
+import { ModelExtraPrices } from "./ModelExtraPrices";
interface ModelHubTableProps {
accessToken: string | null;
@@ -788,6 +789,7 @@ const ModelHubTable: React.FC = ({
: "Not specified"}
+
diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx
index 59da41e7c05..1aaf7760ef2 100644
--- a/ui/litellm-dashboard/src/components/public_model_hub.tsx
+++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx
@@ -55,6 +55,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels";
import { modelUsageExample } from "@/components/AIHub/modelUsageExample";
import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils";
import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata";
+import { ModelExtraPrices } from "@/components/AIHub/ModelExtraPrices";
import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders";
interface PublicModelHubProps {
@@ -884,6 +885,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
: "Not specified"}
+
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 60ef2b0ed76..5f528035fd7 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -38521,7 +38521,7 @@ export interface components {
/** Tpm */
tpm?: number | null;
} & {
- [key: string]: number;
+ [key: string]: unknown;
};
/** ModelInfoDelete */
ModelInfoDelete: {
From e4194f2ff30dfebfe9631cb8764b06480e607c44 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 28 Oct 2026 05:25:23 -0700
Subject: [PATCH 3/8] test(router): list the cost map price helper in the
router coverage checker
---
tests/code_coverage_tests/router_code_coverage.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py
index 853b10ad97b..19481b8a161 100644
--- a/tests/code_coverage_tests/router_code_coverage.py
+++ b/tests/code_coverage_tests/router_code_coverage.py
@@ -172,6 +172,7 @@ ignored_function_names = [
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
+ "_cost_map_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
]
From daf08f1f207bb912e1ff5ca046839efd1cfa8d17 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 28 Oct 2026 06:23:59 -0700
Subject: [PATCH 4/8] fix(router): list only USD price keys on model group
info, show token prices per 1M tokens, and read each deployment's prices in
one pass
---
litellm/router.py | 8 ++---
litellm/types/router.py | 35 ++++++++++---------
.../router_code_coverage.py | 2 +-
.../proxy_server/test_routes_model_info.py | 3 ++
tests/unit/test_router/test_router.py | 18 +++++++---
.../AIHub/ModelExtraPrices.test.tsx | 8 +++--
.../src/components/AIHub/ModelExtraPrices.tsx | 16 +++++++--
7 files changed, 61 insertions(+), 29 deletions(-)
diff --git a/litellm/router.py b/litellm/router.py
index 9e8f958ede8..126c7a5a82c 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -511,10 +511,10 @@ RouterStrategySelector: TypeAlias = (
)
-def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
+def _deployment_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
cost_map_key: Final = model_info.get("key")
entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
- return model_info_prices(entry) if entry is not None else MappingProxyType({})
+ return model_info_prices(model_info if entry is None else {**entry, **model_info})
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
@@ -12592,8 +12592,8 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
- **_cost_map_prices(model_info),
**model_info,
+ **_deployment_prices(model_info),
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
"supported_endpoints": None,
@@ -12627,7 +12627,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
- model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info})
+ model_group_info = model_group_info.with_higher_prices(_deployment_prices(model_info))
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 48434abd35e..bc2fbfc7e31 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -982,13 +982,22 @@ def price_as_float(value: object) -> float | None:
return None
try:
price: Final = float(value)
- except ValueError:
+ except (ValueError, OverflowError):
return None
return price if math.isfinite(price) else None
+def _outprices(price: float, listed: object) -> bool:
+ listed_price: Final = price_as_float(listed)
+ return listed_price is None or price > listed_price
+
+
+def is_usd_price_key(key: str) -> bool:
+ return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key
+
+
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
- candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key)
+ candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if is_usd_price_key(key))
return MappingProxyType({key: price for key, price in candidates if price is not None})
@@ -1042,22 +1051,16 @@ class ModelGroupInfo(LiteLLMBaseModel):
if not isinstance(data, Mapping):
return data
fields: Final = _RAW_FIELDS.validate_python(data)
- declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key}
+ field_names: Final = cls.model_fields.keys()
+ declared: Final = {
+ key: value for key, value in fields.items() if key in field_names and not is_usd_price_key(key)
+ }
return {**declared, **model_info_prices(fields)}
- @property
- def prices(self) -> Mapping[str, float]:
- return model_info_prices({**vars(self), **self.__pydantic_extra__})
-
- def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo":
- current: Final = self.prices
- return self.model_copy(
- update={
- key: price
- for key, price in model_info_prices(model_info).items()
- if key not in current or price > current[key]
- }
- )
+ def with_higher_prices(self, prices: Mapping[str, float]) -> "ModelGroupInfo":
+ listed: Final[Mapping[str, object]] = {**vars(self), **self.__pydantic_extra__}
+ higher: Final = {key: price for key, price in prices.items() if _outprices(price, listed.get(key))}
+ return self.model_copy(update=higher) if higher else self
class AssistantsTypedDict(TypedDict):
diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py
index 19481b8a161..fa6829dc183 100644
--- a/tests/code_coverage_tests/router_code_coverage.py
+++ b/tests/code_coverage_tests/router_code_coverage.py
@@ -172,7 +172,7 @@ ignored_function_names = [
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
- "_cost_map_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
+ "_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
]
diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py
index a2ad764dfa6..68fb8f15abd 100644
--- a/tests/unit/proxy/proxy_server/test_routes_model_info.py
+++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py
@@ -1016,6 +1016,8 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_2K": "Infinity",
"input_cost_per_token": "NaN",
"custom_tags": ["images"],
+ "cost_center": 4021,
+ "input_dbu_cost_per_token": 7e-07,
},
},
{
@@ -1057,6 +1059,7 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_2K": None,
}
assert group["input_cost_per_token"] is None
+ assert (group.get("cost_center"), group.get("input_dbu_cost_per_token")) == (None, None)
def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index a0f29dd3e59..94b60ce2ab2 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2509,7 +2509,6 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
- "input_dbu_cost_per_token": 7e-07,
},
)
router: Final = litellm.Router(
@@ -2523,6 +2522,9 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"output_cost_per_video_per_second": "free",
"output_cost_per_second": "inf",
"search_context_cost_per_query": {"search_context_size_low": 0.01},
+ "input_dbu_cost_per_token": 7e-07,
+ "cost_center": 4021,
+ "cost_allocation_pct": "30",
},
}
for index in deployment_order
@@ -2539,13 +2541,15 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
- "input_dbu_cost_per_token": 7e-07,
}
assert reported.keys().isdisjoint(
{
"output_cost_per_video_per_second",
"output_cost_per_second",
"search_context_cost_per_query",
+ "input_dbu_cost_per_token",
+ "cost_center",
+ "cost_allocation_pct",
"litellm_provider",
"key",
"id",
@@ -2560,14 +2564,18 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon
"output_cost_per_image": 0.08,
"output_cost_per_image_0.5K": 0.06,
"output_cost_per_image_4K": 0.16,
- "input_dbu_cost_per_token": 0.0002,
}
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
- "made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices},
+ "made-up-resolution-model": {
+ "litellm_provider": "openai",
+ "mode": "image_generation",
+ **cost_map_prices,
+ "input_dbu_cost_per_token": 0.0002,
+ },
},
)
_invalidate_model_cost_lowercase_map()
@@ -2585,6 +2593,7 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon
assert info is not None
reported: Final = info.model_dump()
assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices
+ assert "input_dbu_cost_per_token" not in reported
def test_model_group_info_still_validates_from_attributes():
@@ -2945,6 +2954,7 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
("Infinity", None),
("-inf", None),
(float("nan"), None),
+ (10**400, None),
],
)
def test_price_as_float(value: object, expected: float | None):
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
index 97934392993..622ad6c9e21 100644
--- a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
+++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
@@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
import { ModelExtraPrices } from "./ModelExtraPrices";
describe("ModelExtraPrices", () => {
- it("lists every numeric price besides the per-token pair, sorted by name", () => {
+ it("lists every USD price besides the per-token pair, sorted by name, with token prices per 1M tokens", () => {
render(
{
output_cost_per_image: 0.04,
input_cost_per_token_priority: 0.0000025,
output_cost_per_image_4K: 0.16,
+ cache_read_input_token_cost: 0.0000005,
+ input_dbu_cost_per_token: 0.000142858,
+ cost_center: 4021,
output_cost_per_video_per_second: "free",
search_context_cost_per_query: { search_context_size_low: 0.01 },
max_input_tokens: 128000,
@@ -22,7 +25,8 @@ describe("ModelExtraPrices", () => {
const terms = screen.getAllByRole("term").map((term) => term.textContent);
const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent);
expect(terms.map((term, index) => [term, definitions[index]])).toEqual([
- ["input_cost_per_token_priority", "$0.0000025"],
+ ["cache_read_input_token_cost", "$0.5 per 1M tokens"],
+ ["input_cost_per_token_priority", "$2.5 per 1M tokens"],
["output_cost_per_image", "$0.04"],
["output_cost_per_image_4K", "$0.16"],
]);
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
index bb24dd5e115..cbecc76149e 100644
--- a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
+++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
@@ -1,14 +1,26 @@
const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]);
+const TOKENS_PER_LISTED_PRICE = 1_000_000;
+
const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 });
-const isPriceKey = (key: string) => key.includes("cost") && !PRICES_SHOWN_ELSEWHERE.has(key);
+const isUsdPriceKey = (key: string) =>
+ (key.includes("cost_per_") || key.includes("token_cost")) && !key.includes("_dbu_");
+
+const isPerTokenPrice = (key: string) => /_token(_|$)/.test(key);
+
+const isPriceKey = (key: string) => isUsdPriceKey(key) && !PRICES_SHOWN_ELSEWHERE.has(key);
const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
const isExtraPrice = (entry: [string, unknown]): entry is [string, number] =>
isPriceKey(entry[0]) && isFinitePrice(entry[1]);
+const formatPrice = (key: string, price: number) =>
+ isPerTokenPrice(key)
+ ? `$${priceFormat.format(price * TOKENS_PER_LISTED_PRICE)} per 1M tokens`
+ : `$${priceFormat.format(price)}`;
+
export function ModelExtraPrices({ model }: { model: Record }) {
const prices = Object.entries(model)
.filter(isExtraPrice)
@@ -23,7 +35,7 @@ export function ModelExtraPrices({ model }: { model: Record })
{prices.map(([key, price]) => (
{key}
- ${priceFormat.format(price)}
+ {formatPrice(key, price)}
))}
From 075d621f383b5103a250b16d36cda80bea8192a0 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 28 Oct 2026 17:11:55 -0700
Subject: [PATCH 5/8] fix(router): list only billed price keys on model group
info and never price an unresolved deployment by its group name
---
litellm/router.py | 9 ++++--
litellm/types/router.py | 10 +++++-
tests/unit/test_router/test_router.py | 46 +++++++++++++++++++++++++++
3 files changed, 62 insertions(+), 3 deletions(-)
diff --git a/litellm/router.py b/litellm/router.py
index 047a8762097..1f662c1e27a 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -12638,6 +12638,11 @@ class Router:
model_info = None
deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict)
+ deployment_prices = (
+ model_info_prices(_configured_model_info(model))
+ if model_info is None
+ else _deployment_prices(model_info)
+ )
# get llm provider
litellm_model, llm_provider = self._model_group_llm_provider(litellm_params)
@@ -12675,7 +12680,7 @@ class Router:
"providers": [llm_provider],
"description": model_group_description,
**model_info,
- **_deployment_prices(model_info),
+ **deployment_prices,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
"supported_endpoints": None,
@@ -12709,7 +12714,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
- model_group_info = model_group_info.with_higher_prices(_deployment_prices(model_info))
+ model_group_info = model_group_info.with_higher_prices(deployment_prices)
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True
diff --git a/litellm/types/router.py b/litellm/types/router.py
index bc2fbfc7e31..0957708ecf9 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -41,6 +41,7 @@ from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap as _GetModelCostMap
from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router_weights import RouterWeights
@@ -71,6 +72,9 @@ from .utils import (
ModelResponse,
StandardLoggingRoutingDecision,
)
+from .utils import (
+ is_server_derived_pricing_key as _is_server_derived_pricing_key,
+)
from .utils import (
server_owned_wif_litellm_params as _server_owned_wif_litellm_params,
)
@@ -992,8 +996,12 @@ def _outprices(price: float, listed: object) -> bool:
return listed_price is None or price > listed_price
+def _is_billed_price_field(key: str) -> bool:
+ return key in _GetModelCostMap.loaded_model_cost_map_fields() or _is_server_derived_pricing_key(key)
+
+
def is_usd_price_key(key: str) -> bool:
- return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key
+ return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key and _is_billed_price_field(key)
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index ddf690b022d..bdda58f222a 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2509,6 +2509,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
+ "input_cost_per_token_above_999k_tokens": 9e-06,
},
)
router: Final = litellm.Router(
@@ -2525,6 +2526,9 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_dbu_cost_per_token": 7e-07,
"cost_center": 4021,
"cost_allocation_pct": "30",
+ "internal_chargeback_cost_per_token": 0.5,
+ "team_cost_per_seat": 40,
+ "input_cost_per_tokens": 9e-06,
},
}
for index in deployment_order
@@ -2541,9 +2545,13 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
+ "input_cost_per_token_above_999k_tokens": 9e-06,
}
assert reported.keys().isdisjoint(
{
+ "internal_chargeback_cost_per_token",
+ "team_cost_per_seat",
+ "input_cost_per_tokens",
"output_cost_per_video_per_second",
"output_cost_per_second",
"search_context_cost_per_query",
@@ -2596,6 +2604,44 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon
assert "input_dbu_cost_per_token" not in reported
+def test_model_group_info_never_takes_prices_from_a_cost_map_entry_named_like_the_group(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ group_name: Final = "made-up-group-named-like-a-catalog-model"
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": group_name,
+ "litellm_params": {"model": "gemini/made-up-unmapped-model", "api_key": "fake"},
+ "model_info": {"id": "self-hosted-deployment", "input_cost_per_token": 1e-07},
+ }
+ ]
+ )
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ group_name: {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 5e-06,
+ "input_cost_per_token_priority": 1e-05,
+ "output_cost_per_token_batches": 7e-06,
+ },
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+
+ info: Final = router.get_model_group_info(model_group=group_name)
+
+ assert info is not None
+ reported: Final = info.model_dump()
+ assert reported["input_cost_per_token"] == 1e-07
+ assert reported.keys().isdisjoint({"input_cost_per_token_priority", "output_cost_per_token_batches"})
+
+
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
From fae422be30fb035c1cd9f42df3f03e11ed891d59 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 28 Oct 2026 20:44:59 -0700
Subject: [PATCH 6/8] fix(router): list a custom-priced deployment's prices
from its billed entry
A deployment with its own price is billed only from its registered cost
entry, so the group listing now reads that entry for it instead of the
backend model's catalog entry. Budget reservations keep reading only the
group's declared fields, so listing more prices does not change them.
---
.../spend_tracking/budget_reservation.py | 16 ++----
litellm/router.py | 34 +++++++++---
litellm/types/router.py | 12 +++-
.../router_code_coverage.py | 1 +
.../spend_tracking/test_budget_reservation.py | 48 ++++++++++++++++
tests/unit/test_router/test_router.py | 55 ++++++++++++++++++-
6 files changed, 144 insertions(+), 22 deletions(-)
diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py
index 0b91e57f01e..e9f9df90d4b 100644
--- a/litellm/proxy/spend_tracking/budget_reservation.py
+++ b/litellm/proxy/spend_tracking/budget_reservation.py
@@ -44,7 +44,7 @@ from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncreme
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
-from litellm.types.router import DeploymentTypedDict
+from litellm.types.router import DeploymentTypedDict, ModelGroupInfo
@dataclass
@@ -705,14 +705,7 @@ def _model_access_group_counter(group: str, budget: ModelAccessGroupBudget | Non
def _dedupe_tags(tags: list[str]) -> list[str]:
- seen: Final = set()
- deduped_tags: Final = []
- for tag in tags:
- if tag in seen:
- continue
- seen.add(tag)
- deduped_tags.append(tag)
- return deduped_tags
+ return list(dict.fromkeys(tags))
@with_service_target(AUTH_OBJECTS_TARGET)
@@ -1506,6 +1499,9 @@ def _estimate_image_generation_cost(
return cost_per_image * max(n, 1)
+_DECLARED_GROUP_FIELDS: Final = MappingProxyType(dict.fromkeys(ModelGroupInfo.model_fields, True))
+
+
def _get_model_cost_info(
model: str,
llm_router: Router | None,
@@ -1513,7 +1509,7 @@ def _get_model_cost_info(
if llm_router is not None:
model_group_info: Final = llm_router.cached_model_group_info(model)
if model_group_info is not None:
- return model_group_info.model_dump()
+ return model_group_info.model_dump(include=_DECLARED_GROUP_FIELDS)
return dict(litellm.get_model_info(model=model))
diff --git a/litellm/router.py b/litellm/router.py
index 2d582699561..93b7ab7e08f 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -143,6 +143,7 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
@@ -450,6 +451,7 @@ from litellm.types.router import (
SearchToolTypedDict,
TaggedPreRoutingStrategy,
holds_secret_pointer,
+ is_usd_price_key,
model_info_prices,
price_as_float,
)
@@ -545,12 +547,6 @@ RouterStrategySelector: TypeAlias = (
)
-def _deployment_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
- cost_map_key: Final = model_info.get("key")
- entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
- return model_info_prices(model_info if entry is None else {**entry, **model_info})
-
-
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
"""Single owner of the environment-gating rule: a deployment whose model_info names
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
@@ -719,6 +715,28 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
+def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, object]:
+ configured_info: Final = _configured_model_info(deployment)
+ model_id: Final = configured_info.get("id")
+ custom_priced: Final = use_custom_pricing_for_model(
+ {
+ **_LITELLM_PARAMS_ADAPTER.validate_python(deployment.get("litellm_params") or {}),
+ "metadata": {"model_info": dict(configured_info)},
+ }
+ )
+ registered_entry: Final = raw_cost_map_entry(model_id) if custom_priced and isinstance(model_id, str) else None
+ if registered_entry is not None:
+ return registered_entry
+ cost_map_key: Final = model_info.get("key")
+ catalog_entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
+ return model_info if catalog_entry is None else {**catalog_entry, **model_info}
+
+
+def _deployment_prices(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, float]:
+ token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")}
+ return model_info_prices({**_billed_pricing_entry(deployment, model_info), **token_prices})
+
+
def _every_deployment_refused(
deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...]
) -> bool:
@@ -12702,7 +12720,7 @@ class Router:
deployment_prices = (
model_info_prices(_configured_model_info(model))
if model_info is None
- else _deployment_prices(model_info)
+ else _deployment_prices(model, model_info)
)
# get llm provider
@@ -12740,7 +12758,7 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
- **model_info,
+ **{key: value for key, value in model_info.items() if not is_usd_price_key(key)},
**deployment_prices,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 183e12a4f4c..ac60e333ccc 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -7,6 +7,7 @@ import enum
import math
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
+from functools import lru_cache
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@@ -1000,12 +1001,17 @@ def _outprices(price: float, listed: object) -> bool:
return listed_price is None or price > listed_price
-def _is_billed_price_field(key: str) -> bool:
- return key in _GetModelCostMap.loaded_model_cost_map_fields() or _is_server_derived_pricing_key(key)
+@lru_cache(maxsize=4096)
+def _is_usd_price_key_in(key: str, catalog_fields: frozenset[str]) -> bool:
+ return (
+ ("cost_per_" in key or "token_cost" in key)
+ and "_dbu_" not in key
+ and (key in catalog_fields or _is_server_derived_pricing_key(key))
+ )
def is_usd_price_key(key: str) -> bool:
- return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key and _is_billed_price_field(key)
+ return _is_usd_price_key_in(key, _GetModelCostMap.loaded_model_cost_map_fields())
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py
index c4cde058dcd..8ace1f83439 100644
--- a/tests/code_coverage_tests/router_code_coverage.py
+++ b/tests/code_coverage_tests/router_code_coverage.py
@@ -176,6 +176,7 @@ ignored_function_names = [
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
"_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
+ "_billed_pricing_entry", # Read by _deployment_prices, tested through the custom-priced model group info test in test_router.py
"priority_header_update", # Tested through the priority header tests in test_common_request_processing.py
"_drops_params", # Read by split_request_priority and priority_header_update, tested through both
"_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py
diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py
index 024fe5229be..50ca8fcacd5 100644
--- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py
+++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py
@@ -220,6 +220,54 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None:
assert math.isclose(after, before * 1000)
+def _priced_group_router(extra_prices: dict[str, float], mode: str) -> Router:
+ return Router(
+ model_list=[
+ {
+ "model_name": "priced-group",
+ "litellm_params": {
+ "model": "openai/gpt-4o",
+ "api_key": "sk-fake",
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 2e-06,
+ **extra_prices,
+ },
+ "model_info": {"id": "priced-deployment", "mode": mode},
+ }
+ ]
+ )
+
+
+@pytest.mark.parametrize(
+ "extra_prices, mode, route, body",
+ [
+ (
+ {"output_cost_per_reasoning_token": 1e-04},
+ "chat",
+ "/chat/completions",
+ {"model": "priced-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10},
+ ),
+ (
+ {"output_cost_per_image": 0.5},
+ "image_generation",
+ "/images/generations",
+ {"model": "priced-group", "prompt": "a cat", "n": 4},
+ ),
+ ],
+)
+def test_prices_a_group_lists_leave_its_budget_reservation_unchanged(
+ extra_prices: dict[str, float], mode: str, route: str, body: dict[str, object]
+) -> None:
+ without_extra: Final = estimate_request_max_cost(
+ request_body=body, route=route, llm_router=_priced_group_router({}, mode)
+ )
+ with_extra: Final = estimate_request_max_cost(
+ request_body=body, route=route, llm_router=_priced_group_router(extra_prices, mode)
+ )
+
+ assert with_extra == without_extra
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"expiry_offset, expected_max_budget",
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index ad4b677ec10..760e8f6b34b 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -97,7 +97,7 @@ from litellm.llms.base_llm.vector_store.transformation import(
LiteLLMVectorStoreEmbeddingExecutor,
RouterVectorStoreEmbeddingExecutor,
)
-from litellm.types.utils import CallTypes, CredentialItem, Usage
+from litellm.types.utils import CallTypes, CredentialItem, ModelResponse, Usage
from litellm.utils import _invalidate_model_cost_lowercase_map
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE
@@ -2672,6 +2672,59 @@ def test_model_group_info_never_takes_prices_from_a_cost_map_entry_named_like_th
assert reported.keys().isdisjoint({"input_cost_per_token_priority", "output_cost_per_token_batches"})
+def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_bills(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ **litellm.model_cost,
+ "made-up-long-context-model": {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 2e-06,
+ "output_cost_per_token": 8e-06,
+ "input_cost_per_token_above_200k_tokens": 4e-06,
+ },
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "custom-priced-group",
+ "litellm_params": {
+ "model": "openai/made-up-long-context-model",
+ "api_key": "fake",
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 3e-06,
+ },
+ "model_info": {"id": "custom-priced-deployment"},
+ }
+ ]
+ )
+ long_prompt_tokens: Final = 300_000
+
+ response: Final = router.completion(
+ model="custom-priced-group",
+ messages=[{"role": "user", "content": "hi"}],
+ mock_response=ModelResponse(
+ model="made-up-long-context-model",
+ choices=[{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop", "index": 0}],
+ usage=Usage(prompt_tokens=long_prompt_tokens, completion_tokens=0, total_tokens=long_prompt_tokens),
+ ),
+ )
+ info: Final = router.get_model_group_info(model_group="custom-priced-group")
+
+ assert info is not None
+ listed: Final = info.model_dump()
+ assert response._hidden_params["response_cost"] == pytest.approx(long_prompt_tokens * listed["input_cost_per_token"])
+ assert "input_cost_per_token_above_200k_tokens" not in listed
+
+
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
From 23d99b42fea37dd62ce1da37c3babd5b2da42856 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 29 Oct 2026 01:15:07 -0700
Subject: [PATCH 7/8] fix(router): list tier and batch prices at the most any
deployment bills, and scope team deployment prices
A deployment with no flex, priority or batch price still bills those calls at its base rate (half of it for batch input and output), so the group counts that rate through billing's own lookup. A team-owned deployment's prices beyond input and output are listed only for that team's keys.
---
.../litellm_core_utils/llm_cost_calc/utils.py | 17 ++-
litellm/router.py | 105 ++++++++++-----
.../router_code_coverage.py | 10 +-
tests/unit/test_router/test_router.py | 122 ++++++++++++++++++
4 files changed, 220 insertions(+), 34 deletions(-)
diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py
index 4b0b875af6d..edfabf43ad0 100644
--- a/litellm/litellm_core_utils/llm_cost_calc/utils.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py
@@ -910,7 +910,9 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
return 0.0
-def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
+def get_cost_per_unit(
+ model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0
+) -> float | None:
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
cost_per_unit: Final = model_info.get(cost_key)
if isinstance(cost_per_unit, float):
@@ -954,6 +956,19 @@ def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float
_get_cost_per_unit = get_cost_per_unit
+_HALF_PRICED_BATCH_KEYS: Final = frozenset(
+ {f"input_cost_per_token{_BATCH_KEY_SUFFIX}", f"output_cost_per_token{_BATCH_KEY_SUFFIX}"}
+)
+
+
+def billed_rate(model_info: Mapping[str, object], cost_key: str) -> float | None:
+ rate: Final = get_cost_per_unit(model_info, cost_key, default_value=None)
+ if rate is not None or cost_key not in _HALF_PRICED_BATCH_KEYS:
+ return rate
+ standard_rate: Final = get_cost_per_unit(model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None)
+ return None if standard_rate is None else standard_rate / 2
+
+
def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
"""The prices a deployment sets itself, as floats; None when it sets none that parse."""
if model_info is None:
diff --git a/litellm/router.py b/litellm/router.py
index 9d7017aa0de..ec713e55938 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -144,7 +144,7 @@ from litellm.litellm_core_utils.hidden_params import (
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
-from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
+from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES, billed_rate
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
declares_ptu,
@@ -732,9 +732,38 @@ def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[
return model_info if catalog_entry is None else {**catalog_entry, **model_info}
-def _deployment_prices(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, float]:
+def _deployment_price_entry(
+ deployment: Mapping[str, object], model_info: Mapping[str, object] | None
+) -> Mapping[str, object]:
+ if model_info is None:
+ return _configured_model_info(deployment)
token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")}
- return model_info_prices({**_billed_pricing_entry(deployment, model_info), **token_prices})
+ return {**_billed_pricing_entry(deployment, model_info), **token_prices}
+
+
+@dataclass(frozen=True, slots=True)
+class _GroupDeployment:
+ model_info: ModelMapInfo | None
+ price_entry: Mapping[str, object]
+ team_may_use: bool
+
+ def listed_prices(self) -> Mapping[str, float]:
+ prices: Final = model_info_prices(self.price_entry)
+ if self.team_may_use:
+ return prices
+ return MappingProxyType({key: price for key, price in prices.items() if key in ModelGroupInfo.model_fields})
+
+
+def _highest_billed_rate(cost_key: str, price_entries: Sequence[Mapping[str, object]]) -> float | None:
+ rates: Final = (price_as_float(billed_rate(entry, cost_key)) for entry in price_entries)
+ return max((rate for rate in rates if rate is not None), default=None)
+
+
+def _highest_billed_prices(
+ listed_keys: Iterable[str], price_entries: Sequence[Mapping[str, object]]
+) -> Mapping[str, float]:
+ rates: Final = ((key, _highest_billed_rate(key, price_entries)) for key in listed_keys)
+ return MappingProxyType({key: rate for key, rate in rates if rate is not None})
def _every_deployment_refused(
@@ -12652,17 +12681,12 @@ class Router:
(d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)),
None,
)
- for model in model_list:
- is_match = False
- if (
- "model_name" in model
- and model["model_name"] == model_group
- or "model_name" in model
- and self.pattern_router.route(model_group) is not None
- ): # exact match
- is_match = True
-
- if not is_match:
+ group_deployments: Final = tuple(
+ self._group_deployment(model, team_id) if self._is_model_group_member(model, model_group) else None
+ for model in model_list
+ )
+ for model, group_deployment in zip(model_list, group_deployments, strict=True):
+ if group_deployment is None:
continue
# model in model group found #
litellm_params = LiteLLM_Params(**model["litellm_params"])
@@ -12703,22 +12727,9 @@ class Router:
if _deployment_otpm is None:
_deployment_otpm = model_info_dict.get("otpm", None)
- # get model info
- try:
- model_id = model_info_dict.get("id", None)
- if model_id is not None:
- model_info = self.get_deployment_model_info(model_id=model_id, model_name=litellm_params.model)
- else:
- model_info = None
- except Exception:
- model_info = None
-
+ model_info = group_deployment.model_info
deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict)
- deployment_prices = (
- model_info_prices(_configured_model_info(model))
- if model_info is None
- else _deployment_prices(model, model_info)
- )
+ deployment_prices = group_deployment.listed_prices()
# get llm provider
litellm_model, llm_provider = self._model_group_llm_provider(litellm_params)
@@ -12835,7 +12846,7 @@ class Router:
deployment_supported_endpoints(
model_info.get("supported_endpoints"), model, llm_provider
)
- if team_may_use_deployment(_configured_model_info(model).get("team_id"), team_id)
+ if group_deployment.team_may_use
else ()
),
)
@@ -12899,7 +12910,39 @@ class Router:
if configurable_clientside_auth_params is not None:
model_group_info.configurable_clientside_auth_params = configurable_clientside_auth_params
- return model_group_info
+ usable_price_entries: Final = tuple(
+ deployment.price_entry
+ for deployment in group_deployments
+ if deployment is not None and deployment.team_may_use
+ )
+ return model_group_info.with_higher_prices(
+ _highest_billed_prices(model_info_prices(model_group_info.model_dump()), usable_price_entries)
+ )
+
+ return None
+
+ def _is_model_group_member(self, deployment: DeploymentTypedDict, model_group: str) -> bool:
+ return "model_name" in deployment and (
+ deployment["model_name"] == model_group or self.pattern_router.route(model_group) is not None
+ )
+
+ def _deployment_model_info(self, deployment: DeploymentTypedDict) -> ModelMapInfo | None:
+ model_id: Final = _configured_model_info(deployment).get("id")
+ model_name: Final = deployment["litellm_params"].get("model")
+ if not isinstance(model_id, str) or model_name is None:
+ return None
+ try:
+ return self.get_deployment_model_info(model_id=model_id, model_name=model_name)
+ except Exception:
+ return None
+
+ def _group_deployment(self, deployment: DeploymentTypedDict, team_id: str | None) -> _GroupDeployment:
+ model_info: Final = self._deployment_model_info(deployment)
+ return _GroupDeployment(
+ model_info=model_info,
+ price_entry=_deployment_price_entry(deployment, model_info),
+ team_may_use=team_may_use_deployment(_configured_model_info(deployment).get("team_id"), team_id),
+ )
def get_model_group_info(
self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None
diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py
index 8ace1f83439..194e99f4a9b 100644
--- a/tests/code_coverage_tests/router_code_coverage.py
+++ b/tests/code_coverage_tests/router_code_coverage.py
@@ -175,8 +175,14 @@ ignored_function_names = [
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
- "_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
- "_billed_pricing_entry", # Read by _deployment_prices, tested through the custom-priced model group info test in test_router.py
+ "_deployment_price_entry", # Read by _group_deployment, tested through the cost map model group info test in test_router.py
+ "_billed_pricing_entry", # Read by _deployment_price_entry, tested through the custom-priced model group info test in test_router.py
+ "listed_prices", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py
+ "_highest_billed_rate", # Read by _highest_billed_prices, tested through the tier and batch model group info test in test_router.py
+ "_highest_billed_prices", # Read by _set_model_group_info, tested through the tier and batch model group info test in test_router.py
+ "_is_model_group_member", # Read by _set_model_group_info, tested through every model group info test in test_router.py
+ "_deployment_model_info", # Read by _group_deployment, tested through the cost map model group info test in test_router.py
+ "_group_deployment", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py
"priority_header_update", # Tested through the priority header tests in test_common_request_processing.py
"_drops_params", # Read by split_request_priority and priority_header_update, tested through both
"_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index 760e8f6b34b..05166b1c1a2 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2725,6 +2725,128 @@ def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_
assert "input_cost_per_token_above_200k_tokens" not in listed
+def test_model_group_info_lists_tier_and_batch_prices_at_the_most_any_deployment_bills(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ from litellm.cost_calculator import batch_cost_calculator
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ backends: Final = ("made-up-tiered-backend", "made-up-untiered-backend")
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ **litellm.model_cost,
+ backends[0]: {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 4e-06,
+ "input_cost_per_token_flex": 5e-07,
+ "output_cost_per_token_flex": 2e-06,
+ "input_cost_per_token_batches": 4e-07,
+ "output_cost_per_token_batches": 2e-06,
+ },
+ backends[1]: {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 3e-06,
+ "output_cost_per_token": 1.2e-05,
+ },
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "mixed-backend-group",
+ "litellm_params": {"model": f"openai/{backend}", "api_key": "fake"},
+ "model_info": {"id": backend},
+ }
+ for backend in backends
+ ]
+ )
+ tokens: Final = 1_000_000
+ usage: Final = Usage(prompt_tokens=tokens, completion_tokens=tokens, total_tokens=2 * tokens)
+ flex_bills: Final = tuple(
+ litellm.cost_per_token(
+ model=backend,
+ custom_llm_provider="openai",
+ prompt_tokens=tokens,
+ completion_tokens=tokens,
+ service_tier="flex",
+ )
+ for backend in backends
+ )
+ batch_bills: Final = tuple(
+ batch_cost_calculator(usage, model=backend, custom_llm_provider="openai") for backend in backends
+ )
+ highest_bills: Final = (
+ max(prompt_cost for prompt_cost, _ in flex_bills),
+ max(completion_cost for _, completion_cost in flex_bills),
+ max(prompt_cost for prompt_cost, _ in batch_bills),
+ max(completion_cost for _, completion_cost in batch_bills),
+ )
+
+ info: Final = router.get_model_group_info(model_group="mixed-backend-group")
+
+ assert info is not None
+ listed: Final = info.model_dump()
+ listed_keys: Final = (
+ "input_cost_per_token_flex",
+ "output_cost_per_token_flex",
+ "input_cost_per_token_batches",
+ "output_cost_per_token_batches",
+ )
+ assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills)
+
+
+@pytest.mark.parametrize(("team_id", "listed_cache_read_cost"), [(None, 1e-07), ("team-b", 1e-07), ("team-a", 9e-06)])
+def test_model_group_info_lists_a_team_deployment_price_only_to_that_team(
+ monkeypatch: pytest.MonkeyPatch, team_id: str | None, listed_cache_read_cost: float
+):
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ cache_read_costs: Final = {"made-up-shared-backend": 1e-07, "made-up-team-backend": 9e-06}
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ **litellm.model_cost,
+ **{
+ backend: {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 4e-06,
+ "cache_read_input_token_cost": cache_read_cost,
+ }
+ for backend, cache_read_cost in cache_read_costs.items()
+ },
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "team-scoped-group",
+ "litellm_params": {"model": "openai/made-up-shared-backend", "api_key": "fake"},
+ "model_info": {"id": "shared-deployment"},
+ },
+ {
+ "model_name": "team-scoped-group",
+ "litellm_params": {"model": "openai/made-up-team-backend", "api_key": "fake"},
+ "model_info": {"id": "team-a-deployment", "team_id": "team-a"},
+ },
+ ]
+ )
+
+ info: Final = router.get_model_group_info(model_group="team-scoped-group", team_id=team_id)
+
+ assert info is not None
+ assert info.model_dump().get("cache_read_input_token_cost") == listed_cache_read_cost
+
+
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
From 9f7fc215f639bfe5e07bef667e4ee89bd438c1f4 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 29 Oct 2026 04:52:04 -0700
Subject: [PATCH 8/8] fix(router): list cache prices at the input rate billing
charges when a deployment has none
---
.../litellm_core_utils/llm_cost_calc/utils.py | 20 ++++-
tests/unit/test_router/test_router.py | 73 +++++++++++++++++++
2 files changed, 90 insertions(+), 3 deletions(-)
diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py
index edfabf43ad0..5f5d5b4b920 100644
--- a/litellm/litellm_core_utils/llm_cost_calc/utils.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py
@@ -959,14 +959,28 @@ _get_cost_per_unit = get_cost_per_unit
_HALF_PRICED_BATCH_KEYS: Final = frozenset(
{f"input_cost_per_token{_BATCH_KEY_SUFFIX}", f"output_cost_per_token{_BATCH_KEY_SUFFIX}"}
)
+_TIER_KEY_SUFFIXES: Final = ("", *SERVICE_TIER_COST_KEY_SUFFIXES)
+_CACHE_KEYS_BILLED_AT_INPUT_RATE: Final = MappingProxyType(
+ {
+ **{f"cache_read_input_token_cost{suffix}": f"input_cost_per_token{suffix}" for suffix in _TIER_KEY_SUFFIXES},
+ **{
+ f"cache_creation_input_token_cost{suffix}": f"input_cost_per_token{suffix}" for suffix in _TIER_KEY_SUFFIXES
+ },
+ }
+)
def billed_rate(model_info: Mapping[str, object], cost_key: str) -> float | None:
rate: Final = get_cost_per_unit(model_info, cost_key, default_value=None)
- if rate is not None or cost_key not in _HALF_PRICED_BATCH_KEYS:
+ if rate is not None:
return rate
- standard_rate: Final = get_cost_per_unit(model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None)
- return None if standard_rate is None else standard_rate / 2
+ if cost_key in _HALF_PRICED_BATCH_KEYS:
+ standard_rate: Final = get_cost_per_unit(
+ model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None
+ )
+ return None if standard_rate is None else standard_rate / 2
+ input_key: Final = _CACHE_KEYS_BILLED_AT_INPUT_RATE.get(cost_key)
+ return None if input_key is None else get_cost_per_unit(model_info, input_key, default_value=None)
def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index 4acacb6bac2..c1612540224 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -2802,6 +2802,79 @@ def test_model_group_info_lists_tier_and_batch_prices_at_the_most_any_deployment
assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills)
+def test_model_group_info_lists_cache_prices_at_the_most_any_deployment_bills(monkeypatch: pytest.MonkeyPatch):
+ from litellm.utils import _invalidate_model_cost_lowercase_map
+
+ backends: Final = ("made-up-cached-backend", "made-up-uncached-backend")
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ **litellm.model_cost,
+ backends[0]: {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 4e-06,
+ "input_cost_per_token_priority": 2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "cache_read_input_token_cost_priority": 2e-07,
+ "cache_creation_input_token_cost": 1.25e-06,
+ },
+ backends[1]: {
+ "litellm_provider": "openai",
+ "mode": "chat",
+ "input_cost_per_token": 3e-06,
+ "output_cost_per_token": 1.2e-05,
+ "input_cost_per_token_priority": 6e-06,
+ },
+ },
+ )
+ _invalidate_model_cost_lowercase_map()
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "mixed-cache-group",
+ "litellm_params": {"model": f"openai/{backend}", "api_key": "fake"},
+ "model_info": {"id": backend},
+ }
+ for backend in backends
+ ]
+ )
+ tokens: Final = 1_000_000
+
+ def highest_prompt_bill(cache_read_tokens: int, cache_creation_tokens: int, service_tier: str | None) -> float:
+ return max(
+ litellm.cost_per_token(
+ model=backend,
+ custom_llm_provider="openai",
+ prompt_tokens=tokens,
+ completion_tokens=0,
+ cache_read_input_tokens=cache_read_tokens,
+ cache_creation_input_tokens=cache_creation_tokens,
+ service_tier=service_tier,
+ )[0]
+ for backend in backends
+ )
+
+ highest_bills: Final = (
+ highest_prompt_bill(tokens, 0, None),
+ highest_prompt_bill(tokens, 0, "priority"),
+ highest_prompt_bill(0, tokens, None),
+ )
+
+ info: Final = router.get_model_group_info(model_group="mixed-cache-group")
+
+ assert info is not None
+ listed: Final = info.model_dump()
+ listed_keys: Final = (
+ "cache_read_input_token_cost",
+ "cache_read_input_token_cost_priority",
+ "cache_creation_input_token_cost",
+ )
+ assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills)
+
+
@pytest.mark.parametrize(("team_id", "listed_cache_read_cost"), [(None, 1e-07), ("team-b", 1e-07), ("team-a", 9e-06)])
def test_model_group_info_lists_a_team_deployment_price_only_to_that_team(
monkeypatch: pytest.MonkeyPatch, team_id: str | None, listed_cache_read_cost: float