From ec46a981fc14aba24d700b7b97df88e20b98151e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 6 Nov 2026 10:45:44 -0800 Subject: [PATCH] feat(model-hub): show each deployment's full prices in the model hub (#50145) * feat(model-hub): show each deployment's full prices in the model hub The model hub only showed one input and one output price per model, the highest across its deployments. Each model group now lists every deployment's prices, read from the same entry billing charges, including cached tokens, long-context tiers, flex, priority, batch and tiered ranges. Both the logged-in and public hub detail views render them in one table per deployment, labeled by provider only * refactor(ui): move model hub pricing breakdown text onto i18n Route every string the deployment pricing breakdown writes through t() under a new deploymentPricing group in en.json, plus the three hub headings that host it. The English on screen is unchanged * fix(model-hub): show Vertex regional prices and explicit cached reads as billed A Vertex deployment outside the global endpoint now shows its token prices with the regional uplift billing applies at the deployment's location. The explicit cached read price shows per 1M tokens on its own row instead of a raw per-token value. The breakdown also says that adjustments made per request, like off-peak discounts and regional or data residency surcharges, aren't included * fix(model-hub): show prices written as text the way billing reads them A price written in config.yaml as 3e-7 arrives as text. Billing turns it into a number, but the hub dropped it, so a deployment showed no price or the cost map's price instead of its own. The hub now reads numeric text for regular prices and size-based price ranges, and skips text that isn't a number * style(ui): format public model hub test * test(model-hub): name the deployment pricing fixture to stay within the inline object lint budget --- .../litellm_core_utils/llm_cost_calc/utils.py | 2 +- litellm/router_rust/surface/model_info.py | 21 ++ litellm/router_utils/model_group_pricing.py | 69 ++++ litellm/types/router.py | 16 + .../public_endpoints/test_public_endpoints.py | 51 +++ tests/unit/test_router/test_router.py | 275 ++++++++++++++++ .../components/AIHub/ModelHubTable.test.tsx | 31 +- .../src/components/AIHub/ModelHubTable.tsx | 6 + .../components/AIHub/ModelHubTableColumns.tsx | 2 + .../components/PublicModelHubTableColumns.tsx | 2 + .../DeploymentPricingBreakdown.test.tsx | 97 ++++++ .../models/DeploymentPricingBreakdown.tsx | 139 ++++++++ .../models/deploymentPricing.test.ts | 186 +++++++++++ .../molecules/models/deploymentPricing.ts | 305 ++++++++++++++++++ .../publicModelHub/ModelInspector.tsx | 6 + .../PublicAiHub.integration.test.tsx | 25 ++ .../src/components/public_model_hub.test.tsx | 27 ++ .../src/components/public_model_hub.tsx | 8 + ui/litellm-dashboard/src/i18n/locales/en.json | 73 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 31 ++ 20 files changed, 1367 insertions(+), 5 deletions(-) create mode 100644 litellm/router_utils/model_group_pricing.py create mode 100644 ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx create mode 100644 ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx create mode 100644 ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts create mode 100644 ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9ac4f9d5ee8..5eb9cdb11ce 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1464,7 +1464,7 @@ def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | N return 1.0 -def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: +def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float: """ Resolve the per-model uplift multiplier for Vertex AI non-global (regional and multi-region) endpoints. diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index b1649617557..7fd4c22df6d 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -46,6 +46,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( from litellm.litellm_core_utils.batch_utils import deployment_supported_endpoints from litellm.litellm_core_utils.core_helpers import as_str_mapping, coerce_token_limit from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.litellm_logging import resolve_vertex_location_for_cost from litellm.litellm_core_utils.reasoning_effort_capability import ( deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -56,11 +57,13 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage, UsageCounter from litellm.router_python_host import python_host +from litellm.router_utils.model_group_pricing import deployment_pricing from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import build_io_token_rate_limit_headers from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata from litellm.types.router import ( Deployment, DeploymentModelListingInfo, + DeploymentPricing, DeploymentTypedDict, DiscoveredDeploymentModelInfo, LiteLLM_Params, @@ -140,6 +143,11 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _configured_deployment_id(deployment: Mapping[str, object]) -> str | None: + deployment_id: Final = _configured_model_info(deployment).get("id") + return deployment_id if isinstance(deployment_id, str) else None + + def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: """``_operator_max_input_tokens``: the deployment's own limit, skipping one a saved ``/model/info`` response copied from the cost map.""" @@ -187,6 +195,7 @@ class _GroupBuild: # Filled as the lookups arrive. params: dict[int, LiteLLM_Params] = dataclasses.field(default_factory=dict) # mutable-ok: see above info: ModelGroupInfo | None = None + pricing: tuple[DeploymentPricing, ...] = () def _usage_total(values: Sequence[object]) -> int | None: @@ -358,6 +367,7 @@ class HostModelInfo: return None for field, value in (as_str_mapping(answer["assignments"]) or {}).items(): setattr(group.info, field, _group_field(field, wire.from_wire(value))) + group.info.deployment_pricing = group.pricing return group.info def _answer_group_lookup(self, step: Mapping[str, object], group: "_GroupBuild", wire: "_Wire") -> object: @@ -384,6 +394,17 @@ class HostModelInfo: efforts: Final = resolve_supported_reasoning_efforts( model_info, deployment_is_mapped=step["mapped"] is True ) + pricing: Final = deployment_pricing( + llm_provider, + _configured_deployment_id(deployment), + group.params[int(str(step["index"]))].model, + model_info, + resolve_vertex_location_for_cost( + llm_provider, deployment.get("litellm_params"), None, litellm_model + ), + ) + if pricing is not None and pricing not in group.pricing: + group.pricing = (*group.pricing, pricing) # rebind-ok: the merge's state return { "supported_endpoints": wire.to_wire( list( diff --git a/litellm/router_utils/model_group_pricing.py b/litellm/router_utils/model_group_pricing.py new file mode 100644 index 00000000000..afadbc3eb38 --- /dev/null +++ b/litellm/router_utils/model_group_pricing.py @@ -0,0 +1,69 @@ +import math +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import get_vertex_regional_endpoint_uplift +from litellm.types.router import DeploymentPricing, PriceTier + +_RAW_PRICE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def _numeric_text(value: str) -> float | None: + try: + parsed: Final = float(value) + except ValueError: + return None + return parsed if math.isfinite(parsed) else None + + +def _cost_value(value: object) -> float | None: + match value: + case bool(): + return None + case int() | float(): + return float(value) + case str(): + return _numeric_text(value) + case _: + return None + + +def _cost_fields(fields: Mapping[str, object], token_uplift: float) -> Mapping[str, float]: + return { + name: cost * token_uplift if "token" in name else cost + for name, value in fields.items() + if "cost" in name and (cost := _cost_value(value)) is not None + } + + +def _price_tiers(tiered_pricing: object, token_uplift: float) -> tuple[PriceTier, ...]: + try: + return tuple( + PriceTier.model_validate({"range": tier.get("range"), "costs": _cost_fields(tier, token_uplift)}) + for tier in _RAW_PRICE_TIERS.validate_python(tiered_pricing) + ) + except ValidationError: + return () + + +def deployment_pricing( + provider: str, + deployment_id: str | None, + deployment_model: str, + resolved_model_info: Mapping[str, object], + vertex_location: str | None, +) -> DeploymentPricing | None: + """Prices from the entry billing charges this deployment at, so a cost-map variant its own pricing shadows + is never shown. A deployment that declares no pricing is billed at its resolved cost-map entry. Token prices + carry the Vertex regional-endpoint uplift billing applies at the deployment's location.""" + billed_model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=deployment_model) + model_info: Final = resolved_model_info if billed_model_info is None else billed_model_info + token_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + costs: Final = _cost_fields(model_info, token_uplift) + tiers: Final = _price_tiers(model_info.get("tiered_pricing"), token_uplift) + if not costs and not tiers: + return None + return DeploymentPricing(provider=provider, costs=costs, tiered_pricing=tiers) diff --git a/litellm/types/router.py b/litellm/types/router.py index a9f28fbc673..3dcfc694003 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1041,6 +1041,21 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: return MappingProxyType(get_type_hints(model_class)) +class PriceTier(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + range: tuple[float, float] + costs: Mapping[str, float] + + +class DeploymentPricing(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + provider: str + costs: Mapping[str, float] + tiered_pricing: tuple[PriceTier, ...] = () + + class ModelGroupInfo(LiteLLMBaseModel): model_group: str providers: list[str] @@ -1071,6 +1086,7 @@ class ModelGroupInfo(LiteLLMBaseModel): configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None description: str | None = None supported_endpoints: tuple[str, ...] | None = None + deployment_pricing: tuple[DeploymentPricing, ...] = () def __init__(self, **data) -> None: for field_name, field_type in _resolved_annotations(self.__class__).items(): diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d1afcf80ec9..27ba8968200 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -987,6 +987,57 @@ def test_public_model_hub_reports_the_newest_check_of_a_group_not_the_last_row_r app.dependency_overrides.clear() +def test_public_model_hub_lists_each_deployments_prices_without_exposing_the_deployment() -> None: + llm_router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-a", + "api_base": "https://secret-a.internal", + "api_key": "sk-secret-a", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"id": "hub-secret-id-a"}, + }, + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-b", + "api_base": "https://secret-b.internal", + "api_key": "sk-secret-b", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, + }, + "model_info": {"id": "hub-secret-id-b"}, + }, + ] + ) + app: Final = FastAPI() + app.include_router(router) + + with ( + patch("litellm.public_model_groups", ["hub-public"]), + patch("litellm.proxy.proxy_server.llm_router", llm_router), + patch("litellm.proxy.proxy_server.prisma_client", None), + ): + response: Final = TestClient(app).get("/public/model_hub") + + assert response.status_code == 200, response.text + groups: Final = TypeAdapter(list[ModelGroupInfoProxy]).validate_json(response.text) + assert [dict(p.costs) for p in groups[0].deployment_pricing] == [ + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}, + {"input_cost_per_token": 3e-06, "output_cost_per_token": 4e-06}, + ] + leaked: Final = [ + secret + for secret in ("secret-a.internal", "secret-b.internal", "sk-secret", "hub-secret-backend", "hub-secret-id") + if secret in response.text + ] + assert leaked == [], response.text + + # --------------------------------------------------------------------------- # /public/agent_hub # --------------------------------------------------------------------------- diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 271615b18cf..8f8676103b2 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1889,6 +1889,281 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0 +_HUB_PRICES: Final = MappingProxyType( + { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "cache_creation_input_token_cost": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "output_cost_per_token_above_200k_tokens": 8e-06, + "input_cost_per_token_priority": 3e-06, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_batches": 5e-07, + } +) + + +def _cost_map_model(key: str, entry: Mapping[str, object]) -> str: + litellm.register_model( + {key: {"litellm_provider": "openai", "mode": "chat", **entry}}, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + return f"openai/{key}" + + +def test_model_group_info_lists_every_cost_map_price_of_a_deployment(local_model_cost_map: None) -> None: + model: Final = _cost_map_model("hub-pricing-model", {**_HUB_PRICES, "max_input_tokens": 400000}) + router: Final = litellm.Router(model_list=[{"model_name": "hub", "litellm_params": {"model": model}}]) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [(p.provider, dict(p.costs), p.tiered_pricing) for p in group.deployment_pricing] == [ + ("openai", dict(_HUB_PRICES), ()) + ] + + +def test_model_group_info_lists_each_deployments_price_once_next_to_the_unchanged_max( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + override: Final = 3 * _HUB_PRICES["input_cost_per_token"] + router: Final = litellm.Router( + model_list=[ + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://a.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://b.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "input_cost_per_token": override}}, + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [p.costs["input_cost_per_token"] for p in group.deployment_pricing] == [ + _HUB_PRICES["input_cost_per_token"], + override, + ] + assert group.input_cost_per_token == override + assert group.output_cost_per_token == _HUB_PRICES["output_cost_per_token"] + + +def _usage_billed_at(cost_key: str, prompt_tokens: int = 1000) -> tuple[Usage, str | None]: + match cost_key: + case "input_cost_per_token": + return Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens), None + case "output_cost_per_token": + return Usage(prompt_tokens=0, completion_tokens=1000, total_tokens=1000), None + case "cache_read_input_token_cost": + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details={"cached_tokens": 1000}, + ), None + case "cache_creation_input_token_cost": + return Usage( + prompt_tokens=1000, completion_tokens=0, total_tokens=1000, cache_creation_input_tokens=1000 + ), None + case "input_cost_per_token_priority": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "priority" + case "input_cost_per_token_flex": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "flex" + case _: + raise AssertionError(f"no billing scenario for {cost_key}") + + +def _billed_rate(model: str, deployment_id: str, usage: Usage, service_tier: str | None) -> float: + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + service_tier=service_tier, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + +def test_model_group_info_shows_an_overridden_deployment_only_the_prices_billing_charges( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": { + "model": model, + "input_cost_per_token": 3 * _HUB_PRICES["input_cost_per_token"], + "output_cost_per_token": 3 * _HUB_PRICES["output_cost_per_token"], + }, + "model_info": {"id": "hub-override"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + assert {key: _billed_rate(model, "hub-override", *_usage_billed_at(key)) for key in costs} == pytest.approx( + dict(costs) + ) + long_prompt: Final = _usage_billed_at("input_cost_per_token", prompt_tokens=250_000) + assert _billed_rate(model, "hub-override", *long_prompt) == pytest.approx(costs["input_cost_per_token"]) + shadowed: Final = { + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "input_cost_per_token_batches", + } + assert shadowed.isdisjoint(costs), dict(costs) + + +@pytest.mark.parametrize("vertex_location", ["us-east5", "global"]) +def test_model_group_info_shows_a_vertex_deployment_the_token_prices_billing_charges_at_its_location( + local_model_cost_map: None, vertex_location: str +) -> None: + model: Final = "vertex_ai/hub-vertex-model" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "regional_endpoint_uplift_multiplier": 1.1, + } + }, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": vertex_location}, + "model_info": {"id": "hub-vertex"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + + def billed(cost_key: str) -> float: + usage, _ = _usage_billed_at(cost_key) + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="vertex_ai", + router_model_id="hub-vertex", + vertex_location=vertex_location, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + shown: Final = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") + assert {key: costs[key] for key in shown} == pytest.approx({key: billed(key) for key in shown}) + + +def test_model_group_info_lists_tiered_pricing_ranges(local_model_cost_map: None) -> None: + tiers: Final = ( + {"range": [0, 256000], "input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}, + {"range": [256000, 1000000], "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}, + ) + model: Final = _cost_map_model("hub-tiered-model", {"tiered_pricing": list(tiers)}) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "tiered", + "litellm_params": {"model": model, "input_cost_per_token": 1e-06, "output_cost_per_token": 4e-06}, + }, + ] + ) + + group: Final = router.registry().group("tiered") + + assert group is not None + assert [ + [(tier.range, dict(tier.costs)) for tier in pricing.tiered_pricing] for pricing in group.deployment_pricing + ] == [ + [ + ((0.0, 256000.0), {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}), + ((256000.0, 1000000.0), {"input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}), + ], + [], + ] + + +def test_model_group_info_reads_prices_written_as_text_the_way_billing_does(local_model_cost_map: None) -> None: + tier: Final = {"range": [0, 256000], "input_cost_per_token": "5e-08", "output_cost_per_token": "4e-07"} + model: Final = _cost_map_model( + "hub-text-priced-model", {"tiered_pricing": [tier], "cache_read_input_token_cost": "N/A"} + ) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "flat", + "litellm_params": {"model": model, "input_cost_per_token": "3e-7", "output_cost_per_token": "1.2e-6"}, + "model_info": {"id": "hub-text-override"}, + }, + ] + ) + + tiered: Final = router.registry().group("tiered") + flat: Final = router.registry().group("flat") + + assert tiered is not None + assert flat is not None + assert [dict(t.costs) for t in tiered.deployment_pricing[0].tiered_pricing] == [ + {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07} + ] + assert "cache_read_input_token_cost" not in tiered.deployment_pricing[0].costs + flat_costs: Final = flat.deployment_pricing[0].costs + shown: Final = ("input_cost_per_token", "output_cost_per_token") + assert {key: flat_costs[key] for key in shown} == pytest.approx( + {key: _billed_rate(model, "hub-text-override", *_usage_billed_at(key)) for key in shown} + ) + + +def test_model_group_info_lists_custom_prices_of_an_unmapped_deployment_and_skips_an_unpriced_one() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "self-hosted", + "litellm_params": {"model": "openai/hub-unpriced", "api_base": "http://localhost:8000/v1"}, + "model_info": {"cost_tracking_exempt": True}, + }, + { + "model_name": "self-hosted", + "litellm_params": { + "model": "openai/hub-priced", + "api_base": "http://localhost:8001/v1", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"cache_read_input_token_cost": 1e-07}, + }, + ] + ) + + group: Final = router.registry().group("self-hosted") + + assert group is not None + assert [(p.provider, dict(p.costs)) for p in group.deployment_pricing] == [ + ( + "openai", + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06, "cache_read_input_token_cost": 1e-07}, + ) + ] + + def test_model_group_info_description_from_model_info(): """ model_info.description set in the config should surface on ModelGroupInfo, diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 123c7431732..1b30b060ab0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -2,6 +2,7 @@ import * as networking from "@/components/networking"; import userEvent from "@testing-library/user-event"; import { act } from "@testing-library/react"; import type { MCPServerData } from "./MCPHubTableColumns"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import ModelHubTable from "./ModelHubTable"; @@ -374,7 +375,7 @@ describe("ModelHubTable", () => { describe("model details usage example", () => { const openDetails = async ( - model: { model_group: string; providers: string[]; mode: string }, + model: { model_group: string; providers: string[]; mode: string; deployment_pricing?: DeploymentPricing[] }, canOpenPlayground = true, ) => { vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [model] }); @@ -425,6 +426,34 @@ describe("ModelHubTable", () => { expect(screen.queryByRole("link", { name: "Try it in the Playground" })).not.toBeInTheDocument(); }); + it("shows each deployment's own prices, by service tier, in the model details", async () => { + const gpt5OnOpenAIAndAzure = { + model_group: "gpt-5", + providers: ["openai", "azure"], + mode: "chat", + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, input_cost_per_token_priority: 5e-6 }, + tiered_pricing: [], + }, + { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }, + ], + }; + const dialog = await openDetails(gpt5OnOpenAIAndAzure); + + const openai = within(dialog).getByRole("region", { name: "OpenAI pricing" }); + expect(within(openai).getByRole("row", { name: /Input \/ 1M tokens/ })).toHaveTextContent("$2.50$5.00"); + expect(within(dialog).getByRole("region", { name: "Azure pricing" })).toHaveTextContent("$2.75"); + }); + + it("leaves the pricing breakdown out when the proxy sends no deployment prices", async () => { + const dialog = await openDetails({ model_group: "gpt-5", providers: ["openai"], mode: "chat" }); + + expect(within(dialog).getByText("Input Cost per 1M Tokens:")).toBeInTheDocument(); + expect(within(dialog).queryByText("Pricing by Deployment:")).not.toBeInTheDocument(); + }); + it("keeps the chat completions example for a chat model", async () => { const dialog = await openDetails({ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 4d44d0bb809..60259b1f7f0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -57,6 +57,7 @@ import { isDecisionMode, } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import { ModelCustomMetadata } from "./ModelCustomMetadata"; interface ModelHubTableProps { @@ -819,6 +820,11 @@ const ModelHubTable: React.FC = ({

+ {t("aiHub.modelHubTable.modelDialog.pricingByDeployment")}

} + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 2e4af18455f..7f2f3a49d44 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -22,6 +22,7 @@ import { copyToClipboard } from "@/utils/dataUtils"; import { ModelCustomTags, type ModelCustomMetadataValue } from "./ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "./ModelHubProviders"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelHubData { model_group: string; @@ -41,6 +42,7 @@ export interface ModelHubData { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index 08d6b973b87..eb89af22f17 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -11,6 +11,7 @@ import i18n from "@/i18n/i18n"; import { displayMode } from "@/lib/decisionModels"; import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "@/components/AIHub/ModelHubProviders"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelGroupInfo { model_group: string; @@ -32,6 +33,7 @@ export interface ModelGroupInfo { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx new file mode 100644 index 00000000000..08ee05aeb04 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx @@ -0,0 +1,97 @@ +import { render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { DeploymentPricingBreakdown } from "./DeploymentPricingBreakdown"; +import type { DeploymentPricing } from "./deploymentPricing"; + +const OPENAI: DeploymentPricing = { + provider: "openai", + costs: { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + input_cost_per_token_priority: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + output_cost_per_second_720p: 0.05, + }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8, output_cost_per_token: 4e-7 } }], +}; + +const AZURE: DeploymentPricing = { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }; + +const pricesOf = (region: HTMLElement, group: string, charge: string) => { + const rows = within(within(region).getByRole("rowgroup", { name: group })).getAllByRole("row"); + const row = rows.find((candidate) => within(candidate).queryByRole("rowheader", { name: charge })); + expect(row).toBeDefined(); + return within(row as HTMLElement) + .getAllByRole("cell") + .map((cell) => cell.textContent); +}; + +describe("DeploymentPricingBreakdown", () => { + it("renders one block per deployment, headed by its provider, with that deployment's own prices", () => { + render(Pricing} />); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(openai).getByRole("heading", { name: /OpenAI/ })).toBeInTheDocument(); + expect(pricesOf(openai, "Base prices", "Input / 1M tokens")).toEqual(["$2.50", "$5.00"]); + expect(pricesOf(azure, "Base prices", "Input / 1M tokens")).toEqual(["$2.75"]); + expect(within(azure).queryByText(/Blank cells/)).not.toBeInTheDocument(); + expect(screen.getByText(/This model has 2 deployments/)).toBeInTheDocument(); + expect(screen.getAllByText(/Adjustments made per request/)).toHaveLength(1); + }); + + it("lays prices out by service tier column and context threshold group", () => { + render(); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const header = within(openai).getAllByRole("rowgroup")[0]; + expect( + within(header) + .getAllByRole("columnheader") + .map((cell) => cell.textContent), + ).toEqual(["Charge", "Standard", "Priority"]); + expect(pricesOf(openai, "Base prices", "Output / 1M tokens")).toEqual(["$10.00", ""]); + expect(pricesOf(openai, "Above 272K tokens", "Cache read / 1M tokens")).toEqual(["", "$1.00"]); + expect(within(openai).getByText("Blank cells aren't priced separately for this deployment.")).toBeInTheDocument(); + expect(screen.queryByText(/This model has/)).not.toBeInTheDocument(); + }); + + it("shows tiered pricing ranges and keys it could not read", () => { + render(); + + expect(screen.getByText("0–256K tokens: $0.05 input / $0.40 output")).toBeInTheDocument(); + expect(screen.getByText("Output / second 720p")).toBeInTheDocument(); + expect(screen.queryByText("output_cost_per_second_720p")).not.toBeInTheDocument(); + expect(screen.getByText("$0.05")).toBeInTheDocument(); + }); + + it("shows only the request-size ranges for a deployment priced purely by tiers", () => { + render( + , + ); + + expect(screen.getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); + + it.each([ + ["missing", undefined], + ["null", null], + ["empty", []], + ])("renders nothing, heading included, when deployment pricing is %s", (_, deployments) => { + const { container } = render( + Pricing by deployment} />, + ); + + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx new file mode 100644 index 00000000000..3d967e1d43e --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx @@ -0,0 +1,139 @@ +import type * as React from "react"; +import { useTranslation } from "react-i18next"; + +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { cn } from "@/lib/cva.config"; +import { + NUMERIC_CELL_CLASS, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +import { + buildPricingTable, + type DeploymentPricing, + describePricingTier, + formatTokenCount, + type PriceGroup, + SERVICE_TIER_LABELS, + type ServiceTier, +} from "./deploymentPricing"; + +function PriceGroupRows({ group, tiers }: { group: PriceGroup; tiers: readonly ServiceTier[] }) { + const { t } = useTranslation(); + const name = + group.threshold === null + ? t("deploymentPricing.basePrices") + : t("deploymentPricing.aboveThreshold", { threshold: formatTokenCount(group.threshold) }); + return ( + + {group.threshold !== null && ( + + + {name} + + + )} + {group.rows.map((row) => ( + + + {row.label} + + {tiers.map((tier) => ( + + {row.prices[tier]} + + ))} + + ))} + + ); +} + +function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { + const { t } = useTranslation(); + const table = buildPricingTable(deployment.costs); + const tiers = deployment.tiered_pricing; + const providerName = getProviderLogoAndName(deployment.provider).displayName; + return ( +
+

+ + {providerName} +

+ {table.groups.length > 0 && ( + + + + {t("deploymentPricing.chargeColumn")} + {table.tiers.map((tier) => ( + + {t(SERVICE_TIER_LABELS[tier])} + + ))} + + + {table.groups.map((group) => ( + + ))} +
+ )} + {table.hasBlankCells &&

{t("deploymentPricing.blankCells")}

} + {tiers.length > 0 && ( +
+

{t("deploymentPricing.tieredPricing")}

+
    + {tiers.map((tier) => ( +
  • {describePricingTier(tier)}
    • + ))} +
+
+ )} + {table.unknown.length > 0 && ( +
+ {table.unknown.map(({ key, label, price }) => ( +
+
{label}
+
{price}
+
+ ))} +
+ )} +
+ ); +} + +export function DeploymentPricingBreakdown({ + deployments, + heading, + className, +}: { + deployments: readonly DeploymentPricing[] | null | undefined; + heading?: React.ReactNode; + className?: string; +}) { + const { t } = useTranslation(); + if (!deployments || deployments.length === 0) return null; + return ( +
+ {heading} + {deployments.length > 1 && ( +

+ {t("deploymentPricing.deploymentCount", { count: deployments.length })} +

+ )} + {deployments.map((deployment, index) => ( + + ))} +

{t("deploymentPricing.requestAdjustments")}

+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts new file mode 100644 index 00000000000..4e38f55d1b3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; + +import { buildPricingTable, describePricingTier, fallbackCostLabel, parseCostKey } from "./deploymentPricing"; + +describe("parseCostKey", () => { + it.each([ + ["input_cost_per_token", { charge: "input", modality: "", unit: "token", threshold: null, tier: "standard" }], + ["output_cost_per_token_flex", { charge: "output", unit: "token", threshold: null, tier: "flex" }], + ["input_cost_per_token_batches", { charge: "input", tier: "batches" }], + ["cache_read_input_token_cost", { charge: "cache_read", unit: "token", tier: "standard" }], + ["explicit_cache_read_input_token_cost", { charge: "explicit_cache_read", unit: "token", tier: "standard" }], + [ + "cache_read_input_token_cost_above_272k_tokens_priority", + { charge: "cache_read", threshold: 272_000, tier: "priority" }, + ], + [ + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", + { charge: "cache_write", cacheDuration: "12hr", threshold: 272_000, tier: "flex" }, + ], + ["cache_creation_input_token_cost_above_1hr", { charge: "cache_write", cacheDuration: "1hr", threshold: null }], + ["cache_read_input_audio_token_cost_priority", { charge: "cache_read", modality: "audio", tier: "priority" }], + ["input_cost_per_audio_token", { charge: "input", modality: "audio", unit: "token" }], + ["output_cost_per_reasoning_token", { charge: "output", modality: "reasoning", unit: "token" }], + [ + "input_cost_per_audio_per_second_above_128k_tokens", + { charge: "input", modality: "audio", unit: "second", threshold: 128_000 }, + ], + ["output_cost_per_second", { charge: "output", modality: "", unit: "second" }], + ["input_cost_per_image", { charge: "input", modality: "", unit: "image" }], + ["input_cost_per_image_token", { charge: "input", modality: "image", unit: "token" }], + ["output_cost_per_pixel", { unit: "pixel" }], + ["input_cost_per_character", { unit: "character" }], + ["input_cost_per_query", { unit: "query" }], + ["input_cost_per_token_above_200k_tokens_ultrafast", { threshold: 200_000, tier: "ultrafast" }], + ["output_cost_per_token_balanced", { tier: "balanced" }], + ])("reads %s", (key, expected) => { + expect(parseCostKey(key)).toMatchObject({ kind: "price", ...expected }); + }); + + it.each([ + "output_cost_per_second_720p", + "output_cost_per_image_1024", + "input_cost_per_token_cache_hit", + "input_cost_per_token_above_272k_tokens_decisions", + "input_cost_per_video_per_second_above_8s_interval", + "ocr_cost_per_page", + "input_dbu_cost_per_token", + "input_cost_per_text_message", + ])("keeps %s as an unknown key rather than guessing", (key) => { + expect(parseCostKey(key)).toEqual({ kind: "unknown", key }); + }); +}); + +const CONTRACT_COSTS = { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + cache_read_input_token_cost: 1.25e-6, + cache_creation_input_token_cost: 3e-6, + input_cost_per_token_priority: 5e-6, + input_cost_per_token_flex: 1.25e-6, + input_cost_per_token_above_200k_tokens: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + input_cost_per_token_batches: 1.25e-6, + output_cost_per_second: 0.001, + input_cost_per_image: 0.002, +}; + +describe("buildPricingTable", () => { + const table = buildPricingTable(CONTRACT_COSTS); + + it("offers a column per service tier the deployment prices, in a fixed order", () => { + expect(table.tiers).toEqual(["standard", "flex", "priority", "batches"]); + }); + + it("groups prices by context threshold, base prices first", () => { + expect(table.groups.map((group) => group.threshold)).toEqual([null, 200_000, 272_000]); + }); + + it("puts every tier of one charge on one row, token prices scaled to 1M tokens", () => { + expect(table.groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "$2.50", flex: "$1.25", priority: "$5.00", batches: "$1.25" } }, + { label: "Input / image", prices: { standard: "$0.002" } }, + { label: "Cache read / 1M tokens", prices: { standard: "$1.25" } }, + { label: "Cache write / 1M tokens", prices: { standard: "$3.00" } }, + { label: "Output / 1M tokens", prices: { standard: "$10.00" } }, + { label: "Output / second", prices: { standard: "$0.001" } }, + ]); + }); + + it("shows an explicit cached read per 1M tokens on its own row", () => { + const { groups, unknown } = buildPricingTable({ + cache_read_input_token_cost: 1e-7, + explicit_cache_read_input_token_cost: 5e-8, + }); + expect(unknown).toEqual([]); + expect(groups[0].rows).toEqual([ + { label: "Cache read / 1M tokens", prices: { standard: "$0.10" } }, + { label: "Explicit cache read / 1M tokens", prices: { standard: "$0.05" } }, + ]); + }); + + it("answers a cached read on priority above a threshold from its own group", () => { + expect(table.groups[2].rows).toEqual([{ label: "Cache read / 1M tokens", prices: { priority: "$1.00" } }]); + }); + + it("lists keys it cannot read under their raw name with the unscaled price", () => { + const { groups, unknown } = buildPricingTable({ input_cost_per_token: 1e-6, output_cost_per_second_720p: 0.05 }); + expect(unknown).toEqual([{ key: "output_cost_per_second_720p", label: "Output / second 720p", price: "$0.05" }]); + expect(groups.flatMap((group) => group.rows.map((row) => row.label))).toEqual(["Input / 1M tokens"]); + }); + + it("shows a zero price as free and keeps precision for tiny per-unit prices", () => { + const { groups } = buildPricingTable({ input_cost_per_token: 0, output_cost_per_pixel: 1e-8 }); + expect(groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "Free" } }, + { label: "Output / pixel", prices: { standard: "$0.00000001" } }, + ]); + }); + + it("labels modality and cache duration in the row", () => { + const { groups } = buildPricingTable({ + cache_creation_input_token_cost_above_1hr: 6e-6, + input_cost_per_audio_token: 4e-5, + output_cost_per_reasoning_token: 2e-5, + }); + expect(groups[0].rows.map((row) => row.label)).toEqual([ + "Audio input / 1M tokens", + "Cache write (1hr) / 1M tokens", + "Reasoning output / 1M tokens", + ]); + }); + + it("returns an empty table for a deployment with no prices", () => { + const empty = { tiers: [], groups: [], hasBlankCells: false, unknown: [] }; + expect(buildPricingTable({})).toEqual(empty); + }); + + it("flags blank cells only when some charge lacks a price in a tier the deployment offers", () => { + const everyTierPriced = { + input_cost_per_token: 1e-6, + input_cost_per_token_priority: 2e-6, + output_cost_per_token: 3e-6, + output_cost_per_token_priority: 6e-6, + }; + expect(buildPricingTable(CONTRACT_COSTS).hasBlankCells).toBe(true); + expect(buildPricingTable(everyTierPriced).hasBlankCells).toBe(false); + }); +}); + +describe("fallbackCostLabel", () => { + it.each([ + ["google_maps_grounding_cost_per_query", "Google maps grounding / query"], + ["file_search_cost_per_gb_per_day", "File search / gb per day"], + ["cost_per_second", "Cost / second"], + ["output_cost_per_second_720p", "Output / second 720p"], + ["explicit_cache_read_input_token_cost", "Explicit cache read input token"], + ["some_new_price", "Some new price"], + ])("names %s as %s", (key, label) => { + expect(fallbackCostLabel(key)).toBe(label); + }); +}); + +describe("describePricingTier", () => { + it("states the token range and each price per 1M tokens, input before output", () => { + expect( + describePricingTier({ + range: [0, 256_000], + costs: { output_cost_per_token: 4e-7, input_cost_per_token: 5e-8 }, + }), + ).toBe("0–256K tokens: $0.05 input / $0.40 output"); + }); + + it("names extra charges and non-token units", () => { + expect( + describePricingTier({ + range: [256_000, 1_000_000], + costs: { + input_cost_per_token: 2.5e-7, + output_cost_per_reasoning_token: 1.2e-5, + input_cost_per_query: 0.01, + web_search_cost_per_call: 0.02, + }, + }), + ).toBe("256K–1M tokens: $0.25 input / $0.01 input per query / $12.00 reasoning output / $0.02 web search / call"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts new file mode 100644 index 00000000000..337a8900b7d --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts @@ -0,0 +1,305 @@ +import type { ParseKeys } from "i18next"; + +import i18n from "@/i18n/i18n"; +import type { components } from "@/lib/http/schema"; + +export type PricingTier = components["schemas"]["PriceTier"]; +export type DeploymentPricing = components["schemas"]["DeploymentPricing"]; + +export const SERVICE_TIERS = ["standard", "flex", "balanced", "priority", "ultrafast", "batches"] as const; +export type ServiceTier = (typeof SERVICE_TIERS)[number]; + +export const SERVICE_TIER_LABELS: Readonly> = { + standard: "deploymentPricing.tiers.standard", + flex: "deploymentPricing.tiers.flex", + balanced: "deploymentPricing.tiers.balanced", + priority: "deploymentPricing.tiers.priority", + ultrafast: "deploymentPricing.tiers.ultrafast", + batches: "deploymentPricing.tiers.batches", +}; + +const UNITS = ["token", "second", "image", "pixel", "character", "query", "request"] as const; +type Unit = (typeof UNITS)[number]; + +const CHARGES = ["input", "cache_read", "explicit_cache_read", "cache_write", "output"] as const; +type Charge = (typeof CHARGES)[number]; + +const MODALITIES = ["", "text", "image", "audio", "video", "reasoning"] as const; +type Modality = (typeof MODALITIES)[number]; + +export type CostKey = + | { + readonly kind: "price"; + readonly charge: Charge; + readonly modality: Modality; + readonly cacheDuration: string; + readonly unit: Unit; + readonly threshold: number | null; + readonly tier: ServiceTier; + } + | { readonly kind: "unknown"; readonly key: string }; + +const TIER_SUFFIX = /^(.+)_(flex|balanced|priority|ultrafast|batches)$/; +const THRESHOLD_SUFFIX = /^(.+)_above_(\d+)k_tokens$/; +const CACHE_DURATION_SUFFIX = /^(.+)_above_(\d+hr)$/; +const DIRECT_PRICE = /^(input|output)_cost_per_(?:(text|image|audio|video|reasoning)_(?:per_)?)?([a-z]+)$/; +const CACHE_PRICE = /^(explicit_)?cache_(read|creation)_input_(?:(image|audio)_)?token_cost$/; + +const splitSuffix = (key: string, pattern: RegExp): readonly [string, string | null] => { + const match = pattern.exec(key); + return match ? [match[1], match[2]] : [key, null]; +}; + +const isUnit = (value: string): value is Unit => (UNITS as readonly string[]).includes(value); + +const parseBase = (base: string): Pick, "charge" | "modality" | "unit"> | null => { + const direct = DIRECT_PRICE.exec(base); + if (direct) { + const [, charge, modality = "", unit] = direct; + return isUnit(unit) ? { charge: charge as Charge, modality: modality as Modality, unit } : null; + } + const cache = CACHE_PRICE.exec(base); + if (cache) { + const [, explicit, kind, modality = ""] = cache; + if (kind === "creation") + return explicit ? null : { charge: "cache_write", modality: modality as Modality, unit: "token" }; + return { charge: explicit ? "explicit_cache_read" : "cache_read", modality: modality as Modality, unit: "token" }; + } + return null; +}; + +export const parseCostKey = (key: string): CostKey => { + const [withoutTier, tier] = splitSuffix(key, TIER_SUFFIX); + const [withoutThreshold, thousands] = splitSuffix(withoutTier, THRESHOLD_SUFFIX); + const [base, cacheDuration] = splitSuffix(withoutThreshold, CACHE_DURATION_SUFFIX); + const parsed = parseBase(base); + if (!parsed) return { kind: "unknown", key }; + return { + kind: "price", + ...parsed, + cacheDuration: cacheDuration ?? "", + threshold: thousands === null ? null : Number(thousands) * 1000, + tier: (tier ?? "standard") as ServiceTier, + }; +}; + +type Price = Extract; + +type LabelForm = "start" | "inline"; + +const CHARGE_LABELS: Readonly>>> = { + start: { + input: "deploymentPricing.charges.input", + cache_read: "deploymentPricing.charges.cacheRead", + explicit_cache_read: "deploymentPricing.charges.explicitCacheRead", + cache_write: "deploymentPricing.charges.cacheWrite", + output: "deploymentPricing.charges.output", + }, + inline: { + input: "deploymentPricing.chargesInline.input", + cache_read: "deploymentPricing.chargesInline.cacheRead", + explicit_cache_read: "deploymentPricing.chargesInline.explicitCacheRead", + cache_write: "deploymentPricing.chargesInline.cacheWrite", + output: "deploymentPricing.chargesInline.output", + }, +}; + +const MODALITY_LABELS: Readonly, ParseKeys>>>> = { + start: { + text: "deploymentPricing.modalities.text", + image: "deploymentPricing.modalities.image", + audio: "deploymentPricing.modalities.audio", + video: "deploymentPricing.modalities.video", + reasoning: "deploymentPricing.modalities.reasoning", + }, + inline: { + text: "deploymentPricing.modalitiesInline.text", + image: "deploymentPricing.modalitiesInline.image", + audio: "deploymentPricing.modalitiesInline.audio", + video: "deploymentPricing.modalitiesInline.video", + reasoning: "deploymentPricing.modalitiesInline.reasoning", + }, +}; + +const UNIT_LABELS: Readonly> = { + token: "deploymentPricing.units.token", + second: "deploymentPricing.units.second", + image: "deploymentPricing.units.image", + pixel: "deploymentPricing.units.pixel", + character: "deploymentPricing.units.character", + query: "deploymentPricing.units.query", + request: "deploymentPricing.units.request", +}; + +const chargeName = ({ charge, modality }: Price, form: LabelForm): string => + modality + ? i18n.t("deploymentPricing.withModality", { + modality: i18n.t(MODALITY_LABELS[form][modality]), + charge: i18n.t(CHARGE_LABELS.inline[charge]), + }) + : i18n.t(CHARGE_LABELS[form][charge]); + +const chargeLabel = (price: Price, form: LabelForm): string => + price.cacheDuration + ? i18n.t("deploymentPricing.withCacheDuration", { charge: chargeName(price, form), duration: price.cacheDuration }) + : chargeName(price, form); + +const unitLabel = (unit: Unit): string => i18n.t(UNIT_LABELS[unit]); + +const rowLabel = (price: Price): string => + i18n.t("deploymentPricing.rowLabel", { charge: chargeLabel(price, "start"), unit: unitLabel(price.unit) }); + +const rowRank = (price: Price): readonly number[] => [ + CHARGES.indexOf(price.charge), + UNITS.indexOf(price.unit), + MODALITIES.indexOf(price.modality), + parseInt(price.cacheDuration || "0", 10), +]; + +const compareRanks = (a: readonly number[], b: readonly number[]): number => + a.reduce((result, value, index) => result || value - b[index], 0); + +const PRICE_FORMAT_OPTIONS: Intl.NumberFormatOptions = { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 10, + useGrouping: false, +}; + +const PRICE_FORMAT = new Intl.NumberFormat("en-US", PRICE_FORMAT_OPTIONS); + +export const formatRawPrice = (value: number): string => + value === 0 ? i18n.t("deploymentPricing.free") : PRICE_FORMAT.format(value); + +const formatUnitPrice = (value: number, unit: Unit): string => + formatRawPrice(unit === "token" ? value * 1_000_000 : value); + +export interface PriceRow { + readonly label: string; + readonly prices: Readonly>>; +} + +export interface PriceGroup { + readonly threshold: number | null; + readonly rows: readonly PriceRow[]; +} + +export interface UnknownPrice { + readonly key: string; + readonly label: string; + readonly price: string; +} + +export interface PricingTable { + readonly tiers: readonly ServiceTier[]; + readonly groups: readonly PriceGroup[]; + readonly hasBlankCells: boolean; + readonly unknown: readonly UnknownPrice[]; +} + +const PER_UNIT_KEY = /^(.*?)_?cost_per_(.+)$/; +const COST_SUFFIX = /^(.+?)_cost(?:_(.+))?$/; + +const sentenceCase = (text: string): string => { + const words = text.replaceAll("_", " ").trim(); + return words.charAt(0).toUpperCase() + words.slice(1); +}; + +export const fallbackCostLabel = (key: string): string => { + const perUnit = PER_UNIT_KEY.exec(key); + if (perUnit) { + return i18n.t("deploymentPricing.rowLabel", { + charge: perUnit[1] ? sentenceCase(perUnit[1]) : i18n.t("deploymentPricing.fallbackCost"), + unit: perUnit[2].replaceAll("_", " "), + }); + } + const suffixed = COST_SUFFIX.exec(key); + if (suffixed) return sentenceCase([suffixed[1], suffixed[2]].filter(Boolean).join(" ")); + return sentenceCase(key); +}; + +interface ParsedEntry { + readonly key: string; + readonly value: number; + readonly cost: CostKey; +} + +interface PriceEntry extends ParsedEntry { + readonly cost: Price; +} + +const isPriceEntry = (entry: ParsedEntry): entry is PriceEntry => entry.cost.kind === "price"; + +const parseCosts = (costs: Readonly>): readonly ParsedEntry[] => + Object.entries(costs).map(([key, value]) => ({ key, value, cost: parseCostKey(key) })); + +const uniqueBy = (items: readonly T[], keyOf: (item: T) => K): readonly T[] => + items.filter((item, index) => items.findIndex((other) => keyOf(other) === keyOf(item)) === index); + +const byRowRank = (a: PriceEntry, b: PriceEntry): number => compareRanks(rowRank(a.cost), rowRank(b.cost)); + +const groupRows = (entries: readonly PriceEntry[]): readonly PriceRow[] => + [...uniqueBy(entries, ({ cost }) => rowLabel(cost))].sort(byRowRank).map(({ cost }) => ({ + label: rowLabel(cost), + prices: Object.fromEntries( + entries + .filter((other) => rowLabel(other.cost) === rowLabel(cost)) + .map((other) => [other.cost.tier, formatUnitPrice(other.value, other.cost.unit)]), + ), + })); + +export const buildPricingTable = (costs: Readonly>): PricingTable => { + const parsed = parseCosts(costs); + const prices = parsed.filter(isPriceEntry); + const thresholds = [...uniqueBy(prices, ({ cost }) => cost.threshold)] + .map(({ cost }) => cost.threshold) + .sort((a, b) => (a ?? 0) - (b ?? 0)); + const tiers = SERVICE_TIERS.filter((tier) => prices.some(({ cost }) => cost.tier === tier)); + const groups = thresholds.map((threshold) => ({ + threshold, + rows: groupRows(prices.filter(({ cost }) => cost.threshold === threshold)), + })); + return { + tiers, + groups, + hasBlankCells: groups.some(({ rows }) => rows.some(({ prices: row }) => tiers.some((tier) => !(tier in row)))), + unknown: parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => ({ key, label: fallbackCostLabel(key), price: formatRawPrice(value) })), + }; +}; + +export const formatTokenCount = (tokens: number): string => { + if (tokens >= 1_000_000) return `${+(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `${+(tokens / 1_000).toFixed(1)}K`; + return String(tokens); +}; + +const tierPart = ({ cost, value }: PriceEntry): string => { + const charge = chargeLabel(cost, "inline"); + const price = formatUnitPrice(value, cost.unit); + return cost.unit === "token" + ? i18n.t("deploymentPricing.tierPrice", { price, charge }) + : i18n.t("deploymentPricing.tierPricePerUnit", { price, charge, unit: unitLabel(cost.unit) }); +}; + +export const describePricingTier = ({ range: [low, high], costs }: PricingTier): string => { + const parsed = parseCosts(costs); + const parts = [ + ...[...parsed.filter(isPriceEntry)].sort(byRowRank).map(tierPart), + ...parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => + i18n.t("deploymentPricing.tierPrice", { + price: formatRawPrice(value), + charge: fallbackCostLabel(key).toLowerCase(), + }), + ), + ]; + return i18n.t("deploymentPricing.tierRange", { + low: formatTokenCount(low), + high: formatTokenCount(high), + prices: parts.join(" / "), + }); +}; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx index 95901f2bf1d..39495baeea3 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx @@ -4,6 +4,7 @@ import { ChevronDown } from "lucide-react"; import * as React from "react"; import { Trans, useTranslation } from "react-i18next"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; @@ -17,6 +18,7 @@ import { modeLabel, modelCapabilities, providerName, + SECTION_TITLE, wildcardExample, } from "./hubFormat"; import { Badges, CodeBlock, type Fact, Facts, PaneSection, ProviderAvatar, ProviderName } from "./HubParts"; @@ -175,6 +177,10 @@ export function ModelInspector({ model }: { model: ModelGroupInfo }) { )} + {t("publicModelHub.modelInspector.pricingByDeployment")}} + /> {capabilities.length > 0 && ( diff --git a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx index e966836fa3a..cf40a685d76 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx @@ -77,6 +77,13 @@ const MODELS = [ output_cost_per_token: 0, max_input_tokens: 0, description: "General purpose", + deployment_pricing: [ + { + provider: "azure", + costs: { output_cost_per_token: 1e-5, output_cost_per_token_priority: 2e-5 }, + tiered_pricing: [], + }, + ], }, ]; @@ -362,6 +369,24 @@ describe("PublicAiHub", () => { expect(within(pricedRow).queryByRole("cell", { name: "Free" })).not.toBeInTheDocument(); }); + it("shows the open model's prices for each of its deployments", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(await screen.findByRole("link", { name: /gpt-5/ })); + + expect(await screen.findByRole("heading", { name: "Pricing by deployment" })).toBeInTheDocument(); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(azure).getByRole("row", { name: /Output \/ 1M tokens/ })).toHaveTextContent("$10.00$20.00"); + }); + + it("leaves the pricing breakdown out for a model without deployment prices", async () => { + renderWithProviders(, { searchParams: { open: "claude-sonnet-4-5" } }); + + expect(await screen.findByRole("heading", { level: 2, name: "claude-sonnet-4-5" })).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Pricing by deployment" })).not.toBeInTheDocument(); + }); + it("treats an empty open param as nothing opened", async () => { renderWithProviders(, { searchParams: { open: "" } }); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index 515b8230d5b..4081751ca73 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -327,6 +327,33 @@ describe("PublicModelHub", () => { expect(example).toHaveTextContent('model="gpt-4"'); }); + it("shows each deployment's full pricing in the model details", async () => { + respondWith([ + model({ + model_group: "gpt-4", + input_cost_per_token: 2.5e-6, + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, cache_read_input_token_cost_above_272k_tokens_priority: 1e-6 }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8 } }], + }, + ], + }), + ]); + renderHub(); + + fireEvent.click(await screen.findByRole("button", { name: "gpt-4" })); + + const openai = await screen.findByRole("region", { name: "OpenAI pricing" }); + const aboveThreshold = within(openai).getByRole("rowgroup", { name: "Above 272K tokens" }); + expect(within(aboveThreshold).getByRole("row", { name: /Cache read/ })).toHaveTextContent( + "Cache read / 1M tokens$1.00", + ); + expect(within(openai).getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(within(screen.getByRole("dialog")).getByText("$2.5000")).toBeInTheDocument(); + }); + it("filters by several providers at once, and returns to the first page", async () => { const user = userEvent.setup(); respondWith(DEFAULT_MODELS, 300); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index a80bf939165..552ed6fc490 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -56,6 +56,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders"; interface PublicModelHubProps { @@ -954,6 +955,13 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

+ {t("aiHub.publicModelHub.modelDialog.pricingByDeployment")}

+ } + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 919d6cd25cc..3c9dbc99a6b 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -7217,7 +7217,8 @@ "rpm": "Requests per Minute:", "openaiParams": "Supported OpenAI Parameters", "usageExample": "Usage Example", - "copyToClipboard": "Copy to clipboard" + "copyToClipboard": "Copy to clipboard", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7370,7 +7371,8 @@ "usageExample": "Usage Example", "decisionNote": "Decision models answer on /v1/decisions and /v1/systemone, not /chat/completions.", "howToCall": "How to call decision models", - "tryPlayground": "Try it in the Playground" + "tryPlayground": "Try it in the Playground", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7905,7 +7907,8 @@ "title": "Wildcard model", "description": "Call it as , with any model name this provider offers" }, - "capabilities": "Capabilities" + "capabilities": "Capabilities", + "pricingByDeployment": "Pricing by deployment" }, "agentInspector": { "copyName": "Copy agent name", @@ -20300,5 +20303,69 @@ "save": "Save Changes" } } + }, + "deploymentPricing": { + "deploymentCount_one": "This model has {{count}} deployment. Each request is billed at the prices of the deployment that served it.", + "deploymentCount_other": "This model has {{count}} deployments. Each request is billed at the prices of the deployment that served it.", + "requestAdjustments": "These are list prices. Adjustments made per request, like off-peak discounts, regional or data residency surcharges, and some providers' own pricing rules, aren't included.", + "providerPricing": "{{provider}} pricing", + "chargeColumn": "Charge", + "basePrices": "Base prices", + "aboveThreshold": "Above {{threshold}} tokens", + "blankCells": "Blank cells aren't priced separately for this deployment.", + "tieredPricing": "Prices by request size, per 1M tokens", + "tierRange": "{{low}}–{{high}} tokens: {{prices}}", + "tierPrice": "{{price}} {{charge}}", + "tierPricePerUnit": "{{price}} {{charge}} per {{unit}}", + "free": "Free", + "rowLabel": "{{charge}} / {{unit}}", + "withModality": "{{modality}} {{charge}}", + "withCacheDuration": "{{charge}} ({{duration}})", + "fallbackCost": "Cost", + "tiers": { + "standard": "Standard", + "flex": "Flex", + "balanced": "Balanced", + "priority": "Priority", + "ultrafast": "Ultrafast", + "batches": "Batch" + }, + "charges": { + "input": "Input", + "cacheRead": "Cache read", + "explicitCacheRead": "Explicit cache read", + "cacheWrite": "Cache write", + "output": "Output" + }, + "chargesInline": { + "input": "input", + "cacheRead": "cache read", + "explicitCacheRead": "explicit cache read", + "cacheWrite": "cache write", + "output": "output" + }, + "modalities": { + "text": "Text", + "image": "Image", + "audio": "Audio", + "video": "Video", + "reasoning": "Reasoning" + }, + "modalitiesInline": { + "text": "text", + "image": "image", + "audio": "audio", + "video": "video", + "reasoning": "reasoning" + }, + "units": { + "token": "1M tokens", + "second": "second", + "image": "image", + "pixel": "pixel", + "character": "character", + "query": "query", + "request": "request" + } } } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c351e3b1b90..ab1f3c253ea 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32837,6 +32837,20 @@ export interface components { } & { [key: string]: unknown; }; + /** DeploymentPricing */ + DeploymentPricing: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Provider */ + provider: string; + /** + * Tiered Pricing + * @default [] + */ + tiered_pricing: components["schemas"]["PriceTier"][]; + }; /** DeploymentWarning */ DeploymentWarning: { /** Id */ @@ -39687,6 +39701,11 @@ export interface components { } | null; /** Custom Tags */ custom_tags?: string[] | null; + /** + * Deployment Pricing + * @default [] + */ + deployment_pricing: components["schemas"]["DeploymentPricing"][]; /** Description */ description?: string | null; /** Health Checked At */ @@ -42891,6 +42910,18 @@ export interface components { */ version_status: string; }; + /** PriceTier */ + PriceTier: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Range */ + range: [ + number, + number + ]; + }; /** Prompt */ Prompt: { litellm_params: components["schemas"]["PromptLiteLLMParams"];