scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-07 07:58:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 229

fix(router): list only USD price keys on model group info, show token prices per 1M tokens, and read each deployment's prices in one pass

Browse source
This commit is contained in:
mateo-berri 2026-10-28 06:23:59 -07:00
parent e4194f2ff3
commit daf08f1f20
7 changed files with 61 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/router.py
View file

@ -511,10 +511,10 @@ RouterStrategySelector: TypeAlias = (
)
def _cost_map_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
def _deployment_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
cost_map_key: Final = model_info.get("key")
entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
return model_info_prices(entry) if entry is not None else MappingProxyType({})
return model_info_prices(model_info if entry is None else {**entry, **model_info})
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
@ -12592,8 +12592,8 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
**_cost_map_prices(model_info),
**model_info,
**_deployment_prices(model_info),
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
"supported_endpoints": None,
@ -12627,7 +12627,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
model_group_info = model_group_info.with_higher_prices({**_cost_map_prices(model_info), **model_info})
model_group_info = model_group_info.with_higher_prices(_deployment_prices(model_info))
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True

35
litellm/types/router.py
View file

@ -982,13 +982,22 @@ def price_as_float(value: object) -> float | None:
return None
try:
price: Final = float(value)
except ValueError:
except (ValueError, OverflowError):
return None
return price if math.isfinite(price) else None
def _outprices(price: float, listed: object) -> bool:
listed_price: Final = price_as_float(listed)
return listed_price is None or price > listed_price
def is_usd_price_key(key: str) -> bool:
return ("cost_per_" in key or "token_cost" in key) and "_dbu_" not in key
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if "cost" in key)
candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if is_usd_price_key(key))
return MappingProxyType({key: price for key, price in candidates if price is not None})
@ -1042,22 +1051,16 @@ class ModelGroupInfo(LiteLLMBaseModel):
if not isinstance(data, Mapping):
return data
fields: Final = _RAW_FIELDS.validate_python(data)
declared: Final = {key: value for key, value in fields.items() if key in cls.model_fields and "cost" not in key}
field_names: Final = cls.model_fields.keys()
declared: Final = {
key: value for key, value in fields.items() if key in field_names and not is_usd_price_key(key)
}
return {**declared, **model_info_prices(fields)}
@property
def prices(self) -> Mapping[str, float]:
return model_info_prices({**vars(self), **self.__pydantic_extra__})
def with_higher_prices(self, model_info: Mapping[str, object]) -> "ModelGroupInfo":
current: Final = self.prices
return self.model_copy(
update={
key: price
for key, price in model_info_prices(model_info).items()
if key not in current or price > current[key]
}
)
def with_higher_prices(self, prices: Mapping[str, float]) -> "ModelGroupInfo":
listed: Final[Mapping[str, object]] = {**vars(self), **self.__pydantic_extra__}
higher: Final = {key: price for key, price in prices.items() if _outprices(price, listed.get(key))}
return self.model_copy(update=higher) if higher else self
class AssistantsTypedDict(TypedDict):

2
tests/code_coverage_tests/router_code_coverage.py
View file

@ -172,7 +172,7 @@ ignored_function_names = [
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
"_cost_map_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
"_deployment_prices", # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
]

3
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -1016,6 +1016,8 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_2K": "Infinity",
"input_cost_per_token": "NaN",
"custom_tags": ["images"],
"cost_center": 4021,
"input_dbu_cost_per_token": 7e-07,
},
},
{
@ -1057,6 +1059,7 @@ def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_dep
"output_cost_per_image_2K": None,
}
assert group["input_cost_per_token"] is None
assert (group.get("cost_center"), group.get("input_dbu_cost_per_token")) == (None, None)
def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(

18
tests/unit/test_router/test_router.py
View file

@ -2509,7 +2509,6 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
"input_dbu_cost_per_token": 7e-07,
},
)
router: Final = litellm.Router(
@ -2523,6 +2522,9 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"output_cost_per_video_per_second": "free",
"output_cost_per_second": "inf",
"search_context_cost_per_query": {"search_context_size_low": 0.01},
"input_dbu_cost_per_token": 7e-07,
"cost_center": 4021,
"cost_allocation_pct": "30",
},
}
for index in deployment_order
@ -2539,13 +2541,15 @@ def test_model_group_info_reports_every_price_at_the_highest_across_deployments(
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
"input_dbu_cost_per_token": 7e-07,
}
assert reported.keys().isdisjoint(
{
"output_cost_per_video_per_second",
"output_cost_per_second",
"search_context_cost_per_query",
"input_dbu_cost_per_token",
"cost_center",
"cost_allocation_pct",
"litellm_provider",
"key",
"id",
@ -2560,14 +2564,18 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon
"output_cost_per_image": 0.08,
"output_cost_per_image_0.5K": 0.06,
"output_cost_per_image_4K": 0.16,
"input_dbu_cost_per_token": 0.0002,
}
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
"made-up-resolution-model": {"litellm_provider": "openai", "mode": "image_generation", **cost_map_prices},
"made-up-resolution-model": {
"litellm_provider": "openai",
"mode": "image_generation",
**cost_map_prices,
"input_dbu_cost_per_token": 0.0002,
},
},
)
_invalidate_model_cost_lowercase_map()
@ -2585,6 +2593,7 @@ def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(mon
assert info is not None
reported: Final = info.model_dump()
assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices
assert "input_dbu_cost_per_token" not in reported
def test_model_group_info_still_validates_from_attributes():
@ -2945,6 +2954,7 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
("Infinity", None),
("-inf", None),
(float("nan"), None),
(10**400, None),
],
)
def test_price_as_float(value: object, expected: float | None):

8
ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx
View file

@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
import { ModelExtraPrices } from "./ModelExtraPrices";
describe("ModelExtraPrices", () => {
it("lists every numeric price besides the per-token pair, sorted by name", () => {
it("lists every USD price besides the per-token pair, sorted by name, with token prices per 1M tokens", () => {
render(
<ModelExtraPrices
model={{
@ -13,6 +13,9 @@ describe("ModelExtraPrices", () => {
output_cost_per_image: 0.04,
input_cost_per_token_priority: 0.0000025,
output_cost_per_image_4K: 0.16,
cache_read_input_token_cost: 0.0000005,
input_dbu_cost_per_token: 0.000142858,
cost_center: 4021,
output_cost_per_video_per_second: "free",
search_context_cost_per_query: { search_context_size_low: 0.01 },
max_input_tokens: 128000,
@ -22,7 +25,8 @@ describe("ModelExtraPrices", () => {
const terms = screen.getAllByRole("term").map((term) => term.textContent);
const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent);
expect(terms.map((term, index) => [term, definitions[index]])).toEqual([
["input_cost_per_token_priority", "$0.0000025"],
["cache_read_input_token_cost", "$0.5 per 1M tokens"],
["input_cost_per_token_priority", "$2.5 per 1M tokens"],
["output_cost_per_image", "$0.04"],
["output_cost_per_image_4K", "$0.16"],
]);

16
ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx
View file

@ -1,14 +1,26 @@
const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]);
const TOKENS_PER_LISTED_PRICE = 1_000_000;
const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 });
const isPriceKey = (key: string) => key.includes("cost") && !PRICES_SHOWN_ELSEWHERE.has(key);
const isUsdPriceKey = (key: string) =>
(key.includes("cost_per_") || key.includes("token_cost")) && !key.includes("_dbu_");
const isPerTokenPrice = (key: string) => /_token(_|$)/.test(key);
const isPriceKey = (key: string) => isUsdPriceKey(key) && !PRICES_SHOWN_ELSEWHERE.has(key);
const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
const isExtraPrice = (entry: [string, unknown]): entry is [string, number] =>
isPriceKey(entry[0]) && isFinitePrice(entry[1]);
const formatPrice = (key: string, price: number) =>
isPerTokenPrice(key)
? `$${priceFormat.format(price * TOKENS_PER_LISTED_PRICE)} per 1M tokens`
: `$${priceFormat.format(price)}`;
export function ModelExtraPrices({ model }: { model: Record<string, unknown> }) {
const prices = Object.entries(model)
.filter(isExtraPrice)
@ -23,7 +35,7 @@ export function ModelExtraPrices({ model }: { model: Record<string, unknown> })
{prices.map(([key, price]) => (
<div key={key} className="contents">
<dt className="text-muted-foreground">{key}</dt>
<dd>${priceFormat.format(price)}</dd>
<dd>{formatPrice(key, price)}</dd>
</div>
))}
</dl>