scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-07 07:58:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 150

fix(router): list every finite cost map price on /model_group/info and the model hub

Browse source 
Read prices from the deployment's cost map entry as well as its model info, so prices the typed model info leaves out (per-resolution image prices, DBU prices) reach the group. Drop infinite and NaN prices instead of failing the response, and show the extra prices in the dashboard's model hub detail panels.
This commit is contained in:
mateo-berri 2026-10-28 05:09:24 -07:00
parent 286292b6cb
commit ea4e75378c
10 changed files with 173 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/cost_calculator.py
View file

@ -909,7 +909,7 @@ def _bedrock_pricing_info(
def cost_map_key_prices_anything(key: str) -> bool:
entry: Final = _raw_cost_map_entry(key)
entry: Final = raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
@ -2358,7 +2358,7 @@ def _deployment_model_info(
) -> ModelInfo | None:
if not custom_pricing:
return None
registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None
registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None
if registered_deployment_info is not None:
return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id
if litellm_logging_obj is None:
@ -2431,7 +2431,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
def raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@ -2475,7 +2475,7 @@ def pricing_entry_for_cost_calc(
)
if resolved is None:
return None
return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved
return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved
def ocr_cost(

17
litellm/router.py
View file

@ -80,6 +80,7 @@ from litellm.constants import (
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.cost_calculator import raw_cost_map_entry
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
@ -510,6 +511,12 @@ RouterStrategySelector: TypeAlias = (
)
def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
cost_map_key: Final = model_info.get("key")
entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
return model_info_prices(entry) if entry is not None else MappingProxyType({})
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
"""Single owner of the environment-gating rule: a deployment whose model_info names
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
@ -12585,6 +12592,7 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
**_cost_map_prices(model_info),
**model_info,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
@ -12619,14 +12627,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
group_prices = model_group_info.prices
model_group_info = model_group_info.model_copy(
update={
key: price
for key, price in model_info_prices(model_info).items()
if key not in group_prices or price > group_prices[key]
}
)
model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info})
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True

32
litellm/types/router.py
View file

@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
import math
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
@ -70,9 +71,6 @@ from .utils import (
ModelResponse,
StandardLoggingRoutingDecision,
)
from .utils import (
ModelInfo as _CostMapModelInfo,
)
from .utils import (
server_owned_wif_litellm_params as _server_owned_wif_litellm_params,
)
@ -976,11 +974,6 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
return get_type_hints(model_class)
_MODEL_INFO_PRICE_KEYS: Final[frozenset[str]] = frozenset(
key for key in _CostMapModelInfo.__required_keys__ | _CostMapModelInfo.__optional_keys__ if "cost" in key
)
_RAW_FIELDS: Final = TypeAdapter(Mapping[str, object])
@ -988,15 +981,14 @@ def price_as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float | str):
return None
try:
return float(value)
price: Final = float(value)
except ValueError:
return None
return price if math.isfinite(price) else None
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
candidates: Final = (
(key, price_as_float(value)) for key, value in model_info.items() if key in _MODEL_INFO_PRICE_KEYS
)
candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key)
return MappingProxyType({key: price for key, price in candidates if price is not None})
@ -1006,7 +998,7 @@ class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth):
class ModelGroupInfo(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra="allow", json_schema_extra={"additionalProperties": True})
__pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict
model_group: str
@ -1050,13 +1042,23 @@ class ModelGroupInfo(LiteLLMBaseModel):
if not isinstance(data, Mapping):
return data
fields: Final = _RAW_FIELDS.validate_python(data)
declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields}
return {**model_info_prices(fields), **declared}
declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key}
return {**declared, **model_info_prices(fields)}
@property
def prices(self) -> Mapping[str, float]:
return model_info_prices({**vars(self), **self.__pydantic_extra__})
def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo":
current: Final = self.prices
return self.model_copy(
update={
key: price
for key, price in model_info_prices(model_info).items()
if key not in current or price > current[key]
}
)
class AssistantsTypedDict(TypedDict):
custom_llm_provider: Literal["azure", "openai"]

20
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -973,7 +973,11 @@ def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyP
@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool
client: TestClient,
auth_as: Callable[..., AbstractContextManager[object]],
monkeypatch: pytest.MonkeyPatch,
team_id: str | None,
lists_batch: bool,
):
from litellm.proxy._types import LitellmUserRoles
@ -994,7 +998,7 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price(
client, auth_as, monkeypatch
client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy._types import LitellmUserRoles
@ -1008,6 +1012,9 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 2e-05,
"input_cost_per_token_priority": 6e-06,
"output_cost_per_image_4K": 0.16,
"output_cost_per_image_2K": "Infinity",
"input_cost_per_token": "NaN",
"custom_tags": ["images"],
},
},
@ -1038,16 +1045,23 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_token",
"input_cost_per_token_priority",
"input_cost_per_token_above_272k_tokens",
"output_cost_per_image_4K",
"output_cost_per_image_2K",
)
} == {
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"output_cost_per_image_4K": 0.16,
"output_cost_per_image_2K": None,
}
assert group["input_cost_per_token"] is None
def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(client, auth_as, monkeypatch):
def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(
client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy._types import LitellmUserRoles
router: Final = litellm.Router(

53
tests/unit/test_router/test_router.py
View file

@ -2495,7 +2495,7 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar
@pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)])
def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order):
def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order: tuple[int, int]):
deployment_prices: Final = (
{
"output_cost_per_image": 0.5,
@ -2509,6 +2509,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
"input_dbu_cost_per_token": 7e-07,
},
)
router: Final = litellm.Router(
@ -2520,6 +2521,7 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"id": f"tiered-deployment-{index}",
**deployment_prices[index],
"output_cost_per_video_per_second": "free",
"output_cost_per_second": "inf",
"search_context_cost_per_query": {"search_context_size_low": 0.01},
},
}
@ -2537,12 +2539,54 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
"input_dbu_cost_per_token": 7e-07,
}
assert reported.keys().isdisjoint(
{"output_cost_per_video_per_second", "search_context_cost_per_query", "litellm_provider", "key", "id"}
{
"output_cost_per_video_per_second",
"output_cost_per_second",
"search_context_cost_per_query",
"litellm_provider",
"key",
"id",
}
)
def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(monkeypatch: pytest.MonkeyPatch):
from litellm.utils import _invalidate_model_cost_lowercase_map
cost_map_prices: Final = {
"output_cost_per_image": 0.08,
"output_cost_per_image_0.5K": 0.06,
"output_cost_per_image_4K": 0.16,
"input_dbu_cost_per_token": 0.0002,
}
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
"made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices},
},
)
_invalidate_model_cost_lowercase_map()
router: Final = litellm.Router(
model_list=[
{
"model_name": "resolution-group",
"litellm_params": {"model": "openai/made-up-resolution-model", "api_key": "fake"},
}
]
)
info: Final = router.get_model_group_info(model_group="resolution-group")
assert info is not None
reported: Final = info.model_dump()
assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
@ -2898,9 +2942,12 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
("not-a-number", None),
(True, None),
({"input_cost_per_token": 1e-05}, None),
("Infinity", None),
("-inf", None),
(float("nan"), None),
],
)
def test_price_as_float(value, expected):
def test_price_as_float(value: object, expected: float | None):
from litellm.types.router import price_as_float
assert price_as_float(value) == expected

39
ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx Normal file
View file

@ -0,0 +1,39 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
import { ModelExtraPrices } from "./ModelExtraPrices";
describe("ModelExtraPrices", () => {
it("lists every numeric price besides the per-token pair, sorted by name", () => {
render(
<ModelExtraPrices
model={{
model_group: "image-model",
input_cost_per_token: 0.000001,
output_cost_per_token: 0.000002,
output_cost_per_image: 0.04,
input_cost_per_token_priority: 0.0000025,
output_cost_per_image_4K: 0.16,
output_cost_per_video_per_second: "free",
search_context_cost_per_query: { search_context_size_low: 0.01 },
max_input_tokens: 128000,
}}
/>,
);
const terms = screen.getAllByRole("term").map((term) => term.textContent);
const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent);
expect(terms.map((term, index) => [term, definitions[index]])).toEqual([
["input_cost_per_token_priority", "$0.0000025"],
["output_cost_per_image", "$0.04"],
["output_cost_per_image_4K", "$0.16"],
]);
});
it("renders nothing when the model lists only the per-token prices", () => {
const { container } = render(
<ModelExtraPrices
model={{ model_group: "chat-model", input_cost_per_token: 0.000001, output_cost_per_token: 0.000002 }}
/>,
);
expect(container).toBeEmptyDOMElement();
});
});

32
ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx Normal file
View file

@ -0,0 +1,32 @@
const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]);
const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 });
const isPriceKey = (key: string) => key.includes("cost") && !PRICES_SHOWN_ELSEWHERE.has(key);
const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
const isExtraPrice = (entry: [string, unknown]): entry is [string, number] =>
isPriceKey(entry[0]) && isFinitePrice(entry[1]);
export function ModelExtraPrices({ model }: { model: Record<string, unknown> }) {
const prices = Object.entries(model)
.filter(isExtraPrice)
.sort(([a], [b]) => a.localeCompare(b));
if (prices.length === 0) {
return null;
}
return (
<div className="col-span-2">
<p className="font-medium">Other Prices (USD):</p>
<dl className="mt-1 grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1">
{prices.map(([key, price]) => (
<div key={key} className="contents">
<dt className="text-muted-foreground">{key}</dt>
<dd>${priceFormat.format(price)}</dd>
</div>
))}
</dl>
</div>
);
}

2
ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
View file

@ -56,6 +56,7 @@ import {
} from "@/lib/decisionModels";
import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden";
import { ModelCustomMetadata } from "./ModelCustomMetadata";
import { ModelExtraPrices } from "./ModelExtraPrices";
interface ModelHubTableProps {
accessToken: string | null;
@ -788,6 +789,7 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({
: "Not specified"}
</p>
</div>
<ModelExtraPrices model={selectedModel} />
</div>
</div>

2
ui/litellm-dashboard/src/components/public_model_hub.tsx
View file

@ -55,6 +55,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels";
import { modelUsageExample } from "@/components/AIHub/modelUsageExample";
import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils";
import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata";
import { ModelExtraPrices } from "@/components/AIHub/ModelExtraPrices";
import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders";
interface PublicModelHubProps {
@ -884,6 +885,7 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
: "Not specified"}
</p>
</div>
<ModelExtraPrices model={selectedModel} />
</div>
</div>

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -38521,7 +38521,7 @@ export interface components {
/** Tpm */
tpm?: number | null;
} & {
[key: string]: number;
[key: string]: unknown;
};
/** ModelInfoDelete */
ModelInfoDelete: {