diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9ac4f9d5ee8..5eb9cdb11ce 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1464,7 +1464,7 @@ def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | N return 1.0 -def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: +def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float: """ Resolve the per-model uplift multiplier for Vertex AI non-global (regional and multi-region) endpoints. diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index b1649617557..7fd4c22df6d 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -46,6 +46,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( from litellm.litellm_core_utils.batch_utils import deployment_supported_endpoints from litellm.litellm_core_utils.core_helpers import as_str_mapping, coerce_token_limit from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.litellm_logging import resolve_vertex_location_for_cost from litellm.litellm_core_utils.reasoning_effort_capability import ( deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -56,11 +57,13 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage, UsageCounter from litellm.router_python_host import python_host +from litellm.router_utils.model_group_pricing import deployment_pricing from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import build_io_token_rate_limit_headers from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata from litellm.types.router import ( Deployment, DeploymentModelListingInfo, + DeploymentPricing, DeploymentTypedDict, DiscoveredDeploymentModelInfo, LiteLLM_Params, @@ -140,6 +143,11 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _configured_deployment_id(deployment: Mapping[str, object]) -> str | None: + deployment_id: Final = _configured_model_info(deployment).get("id") + return deployment_id if isinstance(deployment_id, str) else None + + def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: """``_operator_max_input_tokens``: the deployment's own limit, skipping one a saved ``/model/info`` response copied from the cost map.""" @@ -187,6 +195,7 @@ class _GroupBuild: # Filled as the lookups arrive. params: dict[int, LiteLLM_Params] = dataclasses.field(default_factory=dict) # mutable-ok: see above info: ModelGroupInfo | None = None + pricing: tuple[DeploymentPricing, ...] = () def _usage_total(values: Sequence[object]) -> int | None: @@ -358,6 +367,7 @@ class HostModelInfo: return None for field, value in (as_str_mapping(answer["assignments"]) or {}).items(): setattr(group.info, field, _group_field(field, wire.from_wire(value))) + group.info.deployment_pricing = group.pricing return group.info def _answer_group_lookup(self, step: Mapping[str, object], group: "_GroupBuild", wire: "_Wire") -> object: @@ -384,6 +394,17 @@ class HostModelInfo: efforts: Final = resolve_supported_reasoning_efforts( model_info, deployment_is_mapped=step["mapped"] is True ) + pricing: Final = deployment_pricing( + llm_provider, + _configured_deployment_id(deployment), + group.params[int(str(step["index"]))].model, + model_info, + resolve_vertex_location_for_cost( + llm_provider, deployment.get("litellm_params"), None, litellm_model + ), + ) + if pricing is not None and pricing not in group.pricing: + group.pricing = (*group.pricing, pricing) # rebind-ok: the merge's state return { "supported_endpoints": wire.to_wire( list( diff --git a/litellm/router_utils/model_group_pricing.py b/litellm/router_utils/model_group_pricing.py new file mode 100644 index 00000000000..afadbc3eb38 --- /dev/null +++ b/litellm/router_utils/model_group_pricing.py @@ -0,0 +1,69 @@ +import math +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import get_vertex_regional_endpoint_uplift +from litellm.types.router import DeploymentPricing, PriceTier + +_RAW_PRICE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def _numeric_text(value: str) -> float | None: + try: + parsed: Final = float(value) + except ValueError: + return None + return parsed if math.isfinite(parsed) else None + + +def _cost_value(value: object) -> float | None: + match value: + case bool(): + return None + case int() | float(): + return float(value) + case str(): + return _numeric_text(value) + case _: + return None + + +def _cost_fields(fields: Mapping[str, object], token_uplift: float) -> Mapping[str, float]: + return { + name: cost * token_uplift if "token" in name else cost + for name, value in fields.items() + if "cost" in name and (cost := _cost_value(value)) is not None + } + + +def _price_tiers(tiered_pricing: object, token_uplift: float) -> tuple[PriceTier, ...]: + try: + return tuple( + PriceTier.model_validate({"range": tier.get("range"), "costs": _cost_fields(tier, token_uplift)}) + for tier in _RAW_PRICE_TIERS.validate_python(tiered_pricing) + ) + except ValidationError: + return () + + +def deployment_pricing( + provider: str, + deployment_id: str | None, + deployment_model: str, + resolved_model_info: Mapping[str, object], + vertex_location: str | None, +) -> DeploymentPricing | None: + """Prices from the entry billing charges this deployment at, so a cost-map variant its own pricing shadows + is never shown. A deployment that declares no pricing is billed at its resolved cost-map entry. Token prices + carry the Vertex regional-endpoint uplift billing applies at the deployment's location.""" + billed_model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=deployment_model) + model_info: Final = resolved_model_info if billed_model_info is None else billed_model_info + token_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + costs: Final = _cost_fields(model_info, token_uplift) + tiers: Final = _price_tiers(model_info.get("tiered_pricing"), token_uplift) + if not costs and not tiers: + return None + return DeploymentPricing(provider=provider, costs=costs, tiered_pricing=tiers) diff --git a/litellm/types/router.py b/litellm/types/router.py index a9f28fbc673..3dcfc694003 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1041,6 +1041,21 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: return MappingProxyType(get_type_hints(model_class)) +class PriceTier(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + range: tuple[float, float] + costs: Mapping[str, float] + + +class DeploymentPricing(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + provider: str + costs: Mapping[str, float] + tiered_pricing: tuple[PriceTier, ...] = () + + class ModelGroupInfo(LiteLLMBaseModel): model_group: str providers: list[str] @@ -1071,6 +1086,7 @@ class ModelGroupInfo(LiteLLMBaseModel): configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None description: str | None = None supported_endpoints: tuple[str, ...] | None = None + deployment_pricing: tuple[DeploymentPricing, ...] = () def __init__(self, **data) -> None: for field_name, field_type in _resolved_annotations(self.__class__).items(): diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d1afcf80ec9..27ba8968200 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -987,6 +987,57 @@ def test_public_model_hub_reports_the_newest_check_of_a_group_not_the_last_row_r app.dependency_overrides.clear() +def test_public_model_hub_lists_each_deployments_prices_without_exposing_the_deployment() -> None: + llm_router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-a", + "api_base": "https://secret-a.internal", + "api_key": "sk-secret-a", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"id": "hub-secret-id-a"}, + }, + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-b", + "api_base": "https://secret-b.internal", + "api_key": "sk-secret-b", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, + }, + "model_info": {"id": "hub-secret-id-b"}, + }, + ] + ) + app: Final = FastAPI() + app.include_router(router) + + with ( + patch("litellm.public_model_groups", ["hub-public"]), + patch("litellm.proxy.proxy_server.llm_router", llm_router), + patch("litellm.proxy.proxy_server.prisma_client", None), + ): + response: Final = TestClient(app).get("/public/model_hub") + + assert response.status_code == 200, response.text + groups: Final = TypeAdapter(list[ModelGroupInfoProxy]).validate_json(response.text) + assert [dict(p.costs) for p in groups[0].deployment_pricing] == [ + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}, + {"input_cost_per_token": 3e-06, "output_cost_per_token": 4e-06}, + ] + leaked: Final = [ + secret + for secret in ("secret-a.internal", "secret-b.internal", "sk-secret", "hub-secret-backend", "hub-secret-id") + if secret in response.text + ] + assert leaked == [], response.text + + # --------------------------------------------------------------------------- # /public/agent_hub # --------------------------------------------------------------------------- diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 271615b18cf..8f8676103b2 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1889,6 +1889,281 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0 +_HUB_PRICES: Final = MappingProxyType( + { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "cache_creation_input_token_cost": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "output_cost_per_token_above_200k_tokens": 8e-06, + "input_cost_per_token_priority": 3e-06, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_batches": 5e-07, + } +) + + +def _cost_map_model(key: str, entry: Mapping[str, object]) -> str: + litellm.register_model( + {key: {"litellm_provider": "openai", "mode": "chat", **entry}}, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + return f"openai/{key}" + + +def test_model_group_info_lists_every_cost_map_price_of_a_deployment(local_model_cost_map: None) -> None: + model: Final = _cost_map_model("hub-pricing-model", {**_HUB_PRICES, "max_input_tokens": 400000}) + router: Final = litellm.Router(model_list=[{"model_name": "hub", "litellm_params": {"model": model}}]) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [(p.provider, dict(p.costs), p.tiered_pricing) for p in group.deployment_pricing] == [ + ("openai", dict(_HUB_PRICES), ()) + ] + + +def test_model_group_info_lists_each_deployments_price_once_next_to_the_unchanged_max( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + override: Final = 3 * _HUB_PRICES["input_cost_per_token"] + router: Final = litellm.Router( + model_list=[ + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://a.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://b.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "input_cost_per_token": override}}, + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [p.costs["input_cost_per_token"] for p in group.deployment_pricing] == [ + _HUB_PRICES["input_cost_per_token"], + override, + ] + assert group.input_cost_per_token == override + assert group.output_cost_per_token == _HUB_PRICES["output_cost_per_token"] + + +def _usage_billed_at(cost_key: str, prompt_tokens: int = 1000) -> tuple[Usage, str | None]: + match cost_key: + case "input_cost_per_token": + return Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens), None + case "output_cost_per_token": + return Usage(prompt_tokens=0, completion_tokens=1000, total_tokens=1000), None + case "cache_read_input_token_cost": + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details={"cached_tokens": 1000}, + ), None + case "cache_creation_input_token_cost": + return Usage( + prompt_tokens=1000, completion_tokens=0, total_tokens=1000, cache_creation_input_tokens=1000 + ), None + case "input_cost_per_token_priority": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "priority" + case "input_cost_per_token_flex": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "flex" + case _: + raise AssertionError(f"no billing scenario for {cost_key}") + + +def _billed_rate(model: str, deployment_id: str, usage: Usage, service_tier: str | None) -> float: + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + service_tier=service_tier, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + +def test_model_group_info_shows_an_overridden_deployment_only_the_prices_billing_charges( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": { + "model": model, + "input_cost_per_token": 3 * _HUB_PRICES["input_cost_per_token"], + "output_cost_per_token": 3 * _HUB_PRICES["output_cost_per_token"], + }, + "model_info": {"id": "hub-override"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + assert {key: _billed_rate(model, "hub-override", *_usage_billed_at(key)) for key in costs} == pytest.approx( + dict(costs) + ) + long_prompt: Final = _usage_billed_at("input_cost_per_token", prompt_tokens=250_000) + assert _billed_rate(model, "hub-override", *long_prompt) == pytest.approx(costs["input_cost_per_token"]) + shadowed: Final = { + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "input_cost_per_token_batches", + } + assert shadowed.isdisjoint(costs), dict(costs) + + +@pytest.mark.parametrize("vertex_location", ["us-east5", "global"]) +def test_model_group_info_shows_a_vertex_deployment_the_token_prices_billing_charges_at_its_location( + local_model_cost_map: None, vertex_location: str +) -> None: + model: Final = "vertex_ai/hub-vertex-model" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "regional_endpoint_uplift_multiplier": 1.1, + } + }, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": vertex_location}, + "model_info": {"id": "hub-vertex"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + + def billed(cost_key: str) -> float: + usage, _ = _usage_billed_at(cost_key) + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="vertex_ai", + router_model_id="hub-vertex", + vertex_location=vertex_location, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + shown: Final = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") + assert {key: costs[key] for key in shown} == pytest.approx({key: billed(key) for key in shown}) + + +def test_model_group_info_lists_tiered_pricing_ranges(local_model_cost_map: None) -> None: + tiers: Final = ( + {"range": [0, 256000], "input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}, + {"range": [256000, 1000000], "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}, + ) + model: Final = _cost_map_model("hub-tiered-model", {"tiered_pricing": list(tiers)}) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "tiered", + "litellm_params": {"model": model, "input_cost_per_token": 1e-06, "output_cost_per_token": 4e-06}, + }, + ] + ) + + group: Final = router.registry().group("tiered") + + assert group is not None + assert [ + [(tier.range, dict(tier.costs)) for tier in pricing.tiered_pricing] for pricing in group.deployment_pricing + ] == [ + [ + ((0.0, 256000.0), {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}), + ((256000.0, 1000000.0), {"input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}), + ], + [], + ] + + +def test_model_group_info_reads_prices_written_as_text_the_way_billing_does(local_model_cost_map: None) -> None: + tier: Final = {"range": [0, 256000], "input_cost_per_token": "5e-08", "output_cost_per_token": "4e-07"} + model: Final = _cost_map_model( + "hub-text-priced-model", {"tiered_pricing": [tier], "cache_read_input_token_cost": "N/A"} + ) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "flat", + "litellm_params": {"model": model, "input_cost_per_token": "3e-7", "output_cost_per_token": "1.2e-6"}, + "model_info": {"id": "hub-text-override"}, + }, + ] + ) + + tiered: Final = router.registry().group("tiered") + flat: Final = router.registry().group("flat") + + assert tiered is not None + assert flat is not None + assert [dict(t.costs) for t in tiered.deployment_pricing[0].tiered_pricing] == [ + {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07} + ] + assert "cache_read_input_token_cost" not in tiered.deployment_pricing[0].costs + flat_costs: Final = flat.deployment_pricing[0].costs + shown: Final = ("input_cost_per_token", "output_cost_per_token") + assert {key: flat_costs[key] for key in shown} == pytest.approx( + {key: _billed_rate(model, "hub-text-override", *_usage_billed_at(key)) for key in shown} + ) + + +def test_model_group_info_lists_custom_prices_of_an_unmapped_deployment_and_skips_an_unpriced_one() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "self-hosted", + "litellm_params": {"model": "openai/hub-unpriced", "api_base": "http://localhost:8000/v1"}, + "model_info": {"cost_tracking_exempt": True}, + }, + { + "model_name": "self-hosted", + "litellm_params": { + "model": "openai/hub-priced", + "api_base": "http://localhost:8001/v1", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"cache_read_input_token_cost": 1e-07}, + }, + ] + ) + + group: Final = router.registry().group("self-hosted") + + assert group is not None + assert [(p.provider, dict(p.costs)) for p in group.deployment_pricing] == [ + ( + "openai", + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06, "cache_read_input_token_cost": 1e-07}, + ) + ] + + def test_model_group_info_description_from_model_info(): """ model_info.description set in the config should surface on ModelGroupInfo, diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 123c7431732..1b30b060ab0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -2,6 +2,7 @@ import * as networking from "@/components/networking"; import userEvent from "@testing-library/user-event"; import { act } from "@testing-library/react"; import type { MCPServerData } from "./MCPHubTableColumns"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import ModelHubTable from "./ModelHubTable"; @@ -374,7 +375,7 @@ describe("ModelHubTable", () => { describe("model details usage example", () => { const openDetails = async ( - model: { model_group: string; providers: string[]; mode: string }, + model: { model_group: string; providers: string[]; mode: string; deployment_pricing?: DeploymentPricing[] }, canOpenPlayground = true, ) => { vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [model] }); @@ -425,6 +426,34 @@ describe("ModelHubTable", () => { expect(screen.queryByRole("link", { name: "Try it in the Playground" })).not.toBeInTheDocument(); }); + it("shows each deployment's own prices, by service tier, in the model details", async () => { + const gpt5OnOpenAIAndAzure = { + model_group: "gpt-5", + providers: ["openai", "azure"], + mode: "chat", + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, input_cost_per_token_priority: 5e-6 }, + tiered_pricing: [], + }, + { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }, + ], + }; + const dialog = await openDetails(gpt5OnOpenAIAndAzure); + + const openai = within(dialog).getByRole("region", { name: "OpenAI pricing" }); + expect(within(openai).getByRole("row", { name: /Input \/ 1M tokens/ })).toHaveTextContent("$2.50$5.00"); + expect(within(dialog).getByRole("region", { name: "Azure pricing" })).toHaveTextContent("$2.75"); + }); + + it("leaves the pricing breakdown out when the proxy sends no deployment prices", async () => { + const dialog = await openDetails({ model_group: "gpt-5", providers: ["openai"], mode: "chat" }); + + expect(within(dialog).getByText("Input Cost per 1M Tokens:")).toBeInTheDocument(); + expect(within(dialog).queryByText("Pricing by Deployment:")).not.toBeInTheDocument(); + }); + it("keeps the chat completions example for a chat model", async () => { const dialog = await openDetails({ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 4d44d0bb809..60259b1f7f0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -57,6 +57,7 @@ import { isDecisionMode, } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import { ModelCustomMetadata } from "./ModelCustomMetadata"; interface ModelHubTableProps { @@ -819,6 +820,11 @@ const ModelHubTable: React.FC = ({

+ {t("aiHub.modelHubTable.modelDialog.pricingByDeployment")}

} + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 2e4af18455f..7f2f3a49d44 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -22,6 +22,7 @@ import { copyToClipboard } from "@/utils/dataUtils"; import { ModelCustomTags, type ModelCustomMetadataValue } from "./ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "./ModelHubProviders"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelHubData { model_group: string; @@ -41,6 +42,7 @@ export interface ModelHubData { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index 08d6b973b87..eb89af22f17 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -11,6 +11,7 @@ import i18n from "@/i18n/i18n"; import { displayMode } from "@/lib/decisionModels"; import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "@/components/AIHub/ModelHubProviders"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelGroupInfo { model_group: string; @@ -32,6 +33,7 @@ export interface ModelGroupInfo { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx new file mode 100644 index 00000000000..08ee05aeb04 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx @@ -0,0 +1,97 @@ +import { render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { DeploymentPricingBreakdown } from "./DeploymentPricingBreakdown"; +import type { DeploymentPricing } from "./deploymentPricing"; + +const OPENAI: DeploymentPricing = { + provider: "openai", + costs: { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + input_cost_per_token_priority: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + output_cost_per_second_720p: 0.05, + }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8, output_cost_per_token: 4e-7 } }], +}; + +const AZURE: DeploymentPricing = { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }; + +const pricesOf = (region: HTMLElement, group: string, charge: string) => { + const rows = within(within(region).getByRole("rowgroup", { name: group })).getAllByRole("row"); + const row = rows.find((candidate) => within(candidate).queryByRole("rowheader", { name: charge })); + expect(row).toBeDefined(); + return within(row as HTMLElement) + .getAllByRole("cell") + .map((cell) => cell.textContent); +}; + +describe("DeploymentPricingBreakdown", () => { + it("renders one block per deployment, headed by its provider, with that deployment's own prices", () => { + render(Pricing} />); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(openai).getByRole("heading", { name: /OpenAI/ })).toBeInTheDocument(); + expect(pricesOf(openai, "Base prices", "Input / 1M tokens")).toEqual(["$2.50", "$5.00"]); + expect(pricesOf(azure, "Base prices", "Input / 1M tokens")).toEqual(["$2.75"]); + expect(within(azure).queryByText(/Blank cells/)).not.toBeInTheDocument(); + expect(screen.getByText(/This model has 2 deployments/)).toBeInTheDocument(); + expect(screen.getAllByText(/Adjustments made per request/)).toHaveLength(1); + }); + + it("lays prices out by service tier column and context threshold group", () => { + render(); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const header = within(openai).getAllByRole("rowgroup")[0]; + expect( + within(header) + .getAllByRole("columnheader") + .map((cell) => cell.textContent), + ).toEqual(["Charge", "Standard", "Priority"]); + expect(pricesOf(openai, "Base prices", "Output / 1M tokens")).toEqual(["$10.00", ""]); + expect(pricesOf(openai, "Above 272K tokens", "Cache read / 1M tokens")).toEqual(["", "$1.00"]); + expect(within(openai).getByText("Blank cells aren't priced separately for this deployment.")).toBeInTheDocument(); + expect(screen.queryByText(/This model has/)).not.toBeInTheDocument(); + }); + + it("shows tiered pricing ranges and keys it could not read", () => { + render(); + + expect(screen.getByText("0–256K tokens: $0.05 input / $0.40 output")).toBeInTheDocument(); + expect(screen.getByText("Output / second 720p")).toBeInTheDocument(); + expect(screen.queryByText("output_cost_per_second_720p")).not.toBeInTheDocument(); + expect(screen.getByText("$0.05")).toBeInTheDocument(); + }); + + it("shows only the request-size ranges for a deployment priced purely by tiers", () => { + render( + , + ); + + expect(screen.getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); + + it.each([ + ["missing", undefined], + ["null", null], + ["empty", []], + ])("renders nothing, heading included, when deployment pricing is %s", (_, deployments) => { + const { container } = render( + Pricing by deployment} />, + ); + + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx new file mode 100644 index 00000000000..3d967e1d43e --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx @@ -0,0 +1,139 @@ +import type * as React from "react"; +import { useTranslation } from "react-i18next"; + +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { cn } from "@/lib/cva.config"; +import { + NUMERIC_CELL_CLASS, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +import { + buildPricingTable, + type DeploymentPricing, + describePricingTier, + formatTokenCount, + type PriceGroup, + SERVICE_TIER_LABELS, + type ServiceTier, +} from "./deploymentPricing"; + +function PriceGroupRows({ group, tiers }: { group: PriceGroup; tiers: readonly ServiceTier[] }) { + const { t } = useTranslation(); + const name = + group.threshold === null + ? t("deploymentPricing.basePrices") + : t("deploymentPricing.aboveThreshold", { threshold: formatTokenCount(group.threshold) }); + return ( + + {group.threshold !== null && ( + + + {name} + + + )} + {group.rows.map((row) => ( + + + {row.label} + + {tiers.map((tier) => ( + + {row.prices[tier]} + + ))} + + ))} + + ); +} + +function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { + const { t } = useTranslation(); + const table = buildPricingTable(deployment.costs); + const tiers = deployment.tiered_pricing; + const providerName = getProviderLogoAndName(deployment.provider).displayName; + return ( +
+

+ + {providerName} +

+ {table.groups.length > 0 && ( + + + + {t("deploymentPricing.chargeColumn")} + {table.tiers.map((tier) => ( + + {t(SERVICE_TIER_LABELS[tier])} + + ))} + + + {table.groups.map((group) => ( + + ))} +
+ )} + {table.hasBlankCells &&

{t("deploymentPricing.blankCells")}

} + {tiers.length > 0 && ( +
+

{t("deploymentPricing.tieredPricing")}

+
    + {tiers.map((tier) => ( +
  • {describePricingTier(tier)}
    • + ))} +
+
+ )} + {table.unknown.length > 0 && ( +
+ {table.unknown.map(({ key, label, price }) => ( +
+
{label}
+
{price}
+
+ ))} +
+ )} +
+ ); +} + +export function DeploymentPricingBreakdown({ + deployments, + heading, + className, +}: { + deployments: readonly DeploymentPricing[] | null | undefined; + heading?: React.ReactNode; + className?: string; +}) { + const { t } = useTranslation(); + if (!deployments || deployments.length === 0) return null; + return ( +
+ {heading} + {deployments.length > 1 && ( +

+ {t("deploymentPricing.deploymentCount", { count: deployments.length })} +

+ )} + {deployments.map((deployment, index) => ( + + ))} +

{t("deploymentPricing.requestAdjustments")}

+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts new file mode 100644 index 00000000000..4e38f55d1b3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; + +import { buildPricingTable, describePricingTier, fallbackCostLabel, parseCostKey } from "./deploymentPricing"; + +describe("parseCostKey", () => { + it.each([ + ["input_cost_per_token", { charge: "input", modality: "", unit: "token", threshold: null, tier: "standard" }], + ["output_cost_per_token_flex", { charge: "output", unit: "token", threshold: null, tier: "flex" }], + ["input_cost_per_token_batches", { charge: "input", tier: "batches" }], + ["cache_read_input_token_cost", { charge: "cache_read", unit: "token", tier: "standard" }], + ["explicit_cache_read_input_token_cost", { charge: "explicit_cache_read", unit: "token", tier: "standard" }], + [ + "cache_read_input_token_cost_above_272k_tokens_priority", + { charge: "cache_read", threshold: 272_000, tier: "priority" }, + ], + [ + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", + { charge: "cache_write", cacheDuration: "12hr", threshold: 272_000, tier: "flex" }, + ], + ["cache_creation_input_token_cost_above_1hr", { charge: "cache_write", cacheDuration: "1hr", threshold: null }], + ["cache_read_input_audio_token_cost_priority", { charge: "cache_read", modality: "audio", tier: "priority" }], + ["input_cost_per_audio_token", { charge: "input", modality: "audio", unit: "token" }], + ["output_cost_per_reasoning_token", { charge: "output", modality: "reasoning", unit: "token" }], + [ + "input_cost_per_audio_per_second_above_128k_tokens", + { charge: "input", modality: "audio", unit: "second", threshold: 128_000 }, + ], + ["output_cost_per_second", { charge: "output", modality: "", unit: "second" }], + ["input_cost_per_image", { charge: "input", modality: "", unit: "image" }], + ["input_cost_per_image_token", { charge: "input", modality: "image", unit: "token" }], + ["output_cost_per_pixel", { unit: "pixel" }], + ["input_cost_per_character", { unit: "character" }], + ["input_cost_per_query", { unit: "query" }], + ["input_cost_per_token_above_200k_tokens_ultrafast", { threshold: 200_000, tier: "ultrafast" }], + ["output_cost_per_token_balanced", { tier: "balanced" }], + ])("reads %s", (key, expected) => { + expect(parseCostKey(key)).toMatchObject({ kind: "price", ...expected }); + }); + + it.each([ + "output_cost_per_second_720p", + "output_cost_per_image_1024", + "input_cost_per_token_cache_hit", + "input_cost_per_token_above_272k_tokens_decisions", + "input_cost_per_video_per_second_above_8s_interval", + "ocr_cost_per_page", + "input_dbu_cost_per_token", + "input_cost_per_text_message", + ])("keeps %s as an unknown key rather than guessing", (key) => { + expect(parseCostKey(key)).toEqual({ kind: "unknown", key }); + }); +}); + +const CONTRACT_COSTS = { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + cache_read_input_token_cost: 1.25e-6, + cache_creation_input_token_cost: 3e-6, + input_cost_per_token_priority: 5e-6, + input_cost_per_token_flex: 1.25e-6, + input_cost_per_token_above_200k_tokens: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + input_cost_per_token_batches: 1.25e-6, + output_cost_per_second: 0.001, + input_cost_per_image: 0.002, +}; + +describe("buildPricingTable", () => { + const table = buildPricingTable(CONTRACT_COSTS); + + it("offers a column per service tier the deployment prices, in a fixed order", () => { + expect(table.tiers).toEqual(["standard", "flex", "priority", "batches"]); + }); + + it("groups prices by context threshold, base prices first", () => { + expect(table.groups.map((group) => group.threshold)).toEqual([null, 200_000, 272_000]); + }); + + it("puts every tier of one charge on one row, token prices scaled to 1M tokens", () => { + expect(table.groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "$2.50", flex: "$1.25", priority: "$5.00", batches: "$1.25" } }, + { label: "Input / image", prices: { standard: "$0.002" } }, + { label: "Cache read / 1M tokens", prices: { standard: "$1.25" } }, + { label: "Cache write / 1M tokens", prices: { standard: "$3.00" } }, + { label: "Output / 1M tokens", prices: { standard: "$10.00" } }, + { label: "Output / second", prices: { standard: "$0.001" } }, + ]); + }); + + it("shows an explicit cached read per 1M tokens on its own row", () => { + const { groups, unknown } = buildPricingTable({ + cache_read_input_token_cost: 1e-7, + explicit_cache_read_input_token_cost: 5e-8, + }); + expect(unknown).toEqual([]); + expect(groups[0].rows).toEqual([ + { label: "Cache read / 1M tokens", prices: { standard: "$0.10" } }, + { label: "Explicit cache read / 1M tokens", prices: { standard: "$0.05" } }, + ]); + }); + + it("answers a cached read on priority above a threshold from its own group", () => { + expect(table.groups[2].rows).toEqual([{ label: "Cache read / 1M tokens", prices: { priority: "$1.00" } }]); + }); + + it("lists keys it cannot read under their raw name with the unscaled price", () => { + const { groups, unknown } = buildPricingTable({ input_cost_per_token: 1e-6, output_cost_per_second_720p: 0.05 }); + expect(unknown).toEqual([{ key: "output_cost_per_second_720p", label: "Output / second 720p", price: "$0.05" }]); + expect(groups.flatMap((group) => group.rows.map((row) => row.label))).toEqual(["Input / 1M tokens"]); + }); + + it("shows a zero price as free and keeps precision for tiny per-unit prices", () => { + const { groups } = buildPricingTable({ input_cost_per_token: 0, output_cost_per_pixel: 1e-8 }); + expect(groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "Free" } }, + { label: "Output / pixel", prices: { standard: "$0.00000001" } }, + ]); + }); + + it("labels modality and cache duration in the row", () => { + const { groups } = buildPricingTable({ + cache_creation_input_token_cost_above_1hr: 6e-6, + input_cost_per_audio_token: 4e-5, + output_cost_per_reasoning_token: 2e-5, + }); + expect(groups[0].rows.map((row) => row.label)).toEqual([ + "Audio input / 1M tokens", + "Cache write (1hr) / 1M tokens", + "Reasoning output / 1M tokens", + ]); + }); + + it("returns an empty table for a deployment with no prices", () => { + const empty = { tiers: [], groups: [], hasBlankCells: false, unknown: [] }; + expect(buildPricingTable({})).toEqual(empty); + }); + + it("flags blank cells only when some charge lacks a price in a tier the deployment offers", () => { + const everyTierPriced = { + input_cost_per_token: 1e-6, + input_cost_per_token_priority: 2e-6, + output_cost_per_token: 3e-6, + output_cost_per_token_priority: 6e-6, + }; + expect(buildPricingTable(CONTRACT_COSTS).hasBlankCells).toBe(true); + expect(buildPricingTable(everyTierPriced).hasBlankCells).toBe(false); + }); +}); + +describe("fallbackCostLabel", () => { + it.each([ + ["google_maps_grounding_cost_per_query", "Google maps grounding / query"], + ["file_search_cost_per_gb_per_day", "File search / gb per day"], + ["cost_per_second", "Cost / second"], + ["output_cost_per_second_720p", "Output / second 720p"], + ["explicit_cache_read_input_token_cost", "Explicit cache read input token"], + ["some_new_price", "Some new price"], + ])("names %s as %s", (key, label) => { + expect(fallbackCostLabel(key)).toBe(label); + }); +}); + +describe("describePricingTier", () => { + it("states the token range and each price per 1M tokens, input before output", () => { + expect( + describePricingTier({ + range: [0, 256_000], + costs: { output_cost_per_token: 4e-7, input_cost_per_token: 5e-8 }, + }), + ).toBe("0–256K tokens: $0.05 input / $0.40 output"); + }); + + it("names extra charges and non-token units", () => { + expect( + describePricingTier({ + range: [256_000, 1_000_000], + costs: { + input_cost_per_token: 2.5e-7, + output_cost_per_reasoning_token: 1.2e-5, + input_cost_per_query: 0.01, + web_search_cost_per_call: 0.02, + }, + }), + ).toBe("256K–1M tokens: $0.25 input / $0.01 input per query / $12.00 reasoning output / $0.02 web search / call"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts new file mode 100644 index 00000000000..337a8900b7d --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts @@ -0,0 +1,305 @@ +import type { ParseKeys } from "i18next"; + +import i18n from "@/i18n/i18n"; +import type { components } from "@/lib/http/schema"; + +export type PricingTier = components["schemas"]["PriceTier"]; +export type DeploymentPricing = components["schemas"]["DeploymentPricing"]; + +export const SERVICE_TIERS = ["standard", "flex", "balanced", "priority", "ultrafast", "batches"] as const; +export type ServiceTier = (typeof SERVICE_TIERS)[number]; + +export const SERVICE_TIER_LABELS: Readonly> = { + standard: "deploymentPricing.tiers.standard", + flex: "deploymentPricing.tiers.flex", + balanced: "deploymentPricing.tiers.balanced", + priority: "deploymentPricing.tiers.priority", + ultrafast: "deploymentPricing.tiers.ultrafast", + batches: "deploymentPricing.tiers.batches", +}; + +const UNITS = ["token", "second", "image", "pixel", "character", "query", "request"] as const; +type Unit = (typeof UNITS)[number]; + +const CHARGES = ["input", "cache_read", "explicit_cache_read", "cache_write", "output"] as const; +type Charge = (typeof CHARGES)[number]; + +const MODALITIES = ["", "text", "image", "audio", "video", "reasoning"] as const; +type Modality = (typeof MODALITIES)[number]; + +export type CostKey = + | { + readonly kind: "price"; + readonly charge: Charge; + readonly modality: Modality; + readonly cacheDuration: string; + readonly unit: Unit; + readonly threshold: number | null; + readonly tier: ServiceTier; + } + | { readonly kind: "unknown"; readonly key: string }; + +const TIER_SUFFIX = /^(.+)_(flex|balanced|priority|ultrafast|batches)$/; +const THRESHOLD_SUFFIX = /^(.+)_above_(\d+)k_tokens$/; +const CACHE_DURATION_SUFFIX = /^(.+)_above_(\d+hr)$/; +const DIRECT_PRICE = /^(input|output)_cost_per_(?:(text|image|audio|video|reasoning)_(?:per_)?)?([a-z]+)$/; +const CACHE_PRICE = /^(explicit_)?cache_(read|creation)_input_(?:(image|audio)_)?token_cost$/; + +const splitSuffix = (key: string, pattern: RegExp): readonly [string, string | null] => { + const match = pattern.exec(key); + return match ? [match[1], match[2]] : [key, null]; +}; + +const isUnit = (value: string): value is Unit => (UNITS as readonly string[]).includes(value); + +const parseBase = (base: string): Pick, "charge" | "modality" | "unit"> | null => { + const direct = DIRECT_PRICE.exec(base); + if (direct) { + const [, charge, modality = "", unit] = direct; + return isUnit(unit) ? { charge: charge as Charge, modality: modality as Modality, unit } : null; + } + const cache = CACHE_PRICE.exec(base); + if (cache) { + const [, explicit, kind, modality = ""] = cache; + if (kind === "creation") + return explicit ? null : { charge: "cache_write", modality: modality as Modality, unit: "token" }; + return { charge: explicit ? "explicit_cache_read" : "cache_read", modality: modality as Modality, unit: "token" }; + } + return null; +}; + +export const parseCostKey = (key: string): CostKey => { + const [withoutTier, tier] = splitSuffix(key, TIER_SUFFIX); + const [withoutThreshold, thousands] = splitSuffix(withoutTier, THRESHOLD_SUFFIX); + const [base, cacheDuration] = splitSuffix(withoutThreshold, CACHE_DURATION_SUFFIX); + const parsed = parseBase(base); + if (!parsed) return { kind: "unknown", key }; + return { + kind: "price", + ...parsed, + cacheDuration: cacheDuration ?? "", + threshold: thousands === null ? null : Number(thousands) * 1000, + tier: (tier ?? "standard") as ServiceTier, + }; +}; + +type Price = Extract; + +type LabelForm = "start" | "inline"; + +const CHARGE_LABELS: Readonly>>> = { + start: { + input: "deploymentPricing.charges.input", + cache_read: "deploymentPricing.charges.cacheRead", + explicit_cache_read: "deploymentPricing.charges.explicitCacheRead", + cache_write: "deploymentPricing.charges.cacheWrite", + output: "deploymentPricing.charges.output", + }, + inline: { + input: "deploymentPricing.chargesInline.input", + cache_read: "deploymentPricing.chargesInline.cacheRead", + explicit_cache_read: "deploymentPricing.chargesInline.explicitCacheRead", + cache_write: "deploymentPricing.chargesInline.cacheWrite", + output: "deploymentPricing.chargesInline.output", + }, +}; + +const MODALITY_LABELS: Readonly, ParseKeys>>>> = { + start: { + text: "deploymentPricing.modalities.text", + image: "deploymentPricing.modalities.image", + audio: "deploymentPricing.modalities.audio", + video: "deploymentPricing.modalities.video", + reasoning: "deploymentPricing.modalities.reasoning", + }, + inline: { + text: "deploymentPricing.modalitiesInline.text", + image: "deploymentPricing.modalitiesInline.image", + audio: "deploymentPricing.modalitiesInline.audio", + video: "deploymentPricing.modalitiesInline.video", + reasoning: "deploymentPricing.modalitiesInline.reasoning", + }, +}; + +const UNIT_LABELS: Readonly> = { + token: "deploymentPricing.units.token", + second: "deploymentPricing.units.second", + image: "deploymentPricing.units.image", + pixel: "deploymentPricing.units.pixel", + character: "deploymentPricing.units.character", + query: "deploymentPricing.units.query", + request: "deploymentPricing.units.request", +}; + +const chargeName = ({ charge, modality }: Price, form: LabelForm): string => + modality + ? i18n.t("deploymentPricing.withModality", { + modality: i18n.t(MODALITY_LABELS[form][modality]), + charge: i18n.t(CHARGE_LABELS.inline[charge]), + }) + : i18n.t(CHARGE_LABELS[form][charge]); + +const chargeLabel = (price: Price, form: LabelForm): string => + price.cacheDuration + ? i18n.t("deploymentPricing.withCacheDuration", { charge: chargeName(price, form), duration: price.cacheDuration }) + : chargeName(price, form); + +const unitLabel = (unit: Unit): string => i18n.t(UNIT_LABELS[unit]); + +const rowLabel = (price: Price): string => + i18n.t("deploymentPricing.rowLabel", { charge: chargeLabel(price, "start"), unit: unitLabel(price.unit) }); + +const rowRank = (price: Price): readonly number[] => [ + CHARGES.indexOf(price.charge), + UNITS.indexOf(price.unit), + MODALITIES.indexOf(price.modality), + parseInt(price.cacheDuration || "0", 10), +]; + +const compareRanks = (a: readonly number[], b: readonly number[]): number => + a.reduce((result, value, index) => result || value - b[index], 0); + +const PRICE_FORMAT_OPTIONS: Intl.NumberFormatOptions = { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 10, + useGrouping: false, +}; + +const PRICE_FORMAT = new Intl.NumberFormat("en-US", PRICE_FORMAT_OPTIONS); + +export const formatRawPrice = (value: number): string => + value === 0 ? i18n.t("deploymentPricing.free") : PRICE_FORMAT.format(value); + +const formatUnitPrice = (value: number, unit: Unit): string => + formatRawPrice(unit === "token" ? value * 1_000_000 : value); + +export interface PriceRow { + readonly label: string; + readonly prices: Readonly>>; +} + +export interface PriceGroup { + readonly threshold: number | null; + readonly rows: readonly PriceRow[]; +} + +export interface UnknownPrice { + readonly key: string; + readonly label: string; + readonly price: string; +} + +export interface PricingTable { + readonly tiers: readonly ServiceTier[]; + readonly groups: readonly PriceGroup[]; + readonly hasBlankCells: boolean; + readonly unknown: readonly UnknownPrice[]; +} + +const PER_UNIT_KEY = /^(.*?)_?cost_per_(.+)$/; +const COST_SUFFIX = /^(.+?)_cost(?:_(.+))?$/; + +const sentenceCase = (text: string): string => { + const words = text.replaceAll("_", " ").trim(); + return words.charAt(0).toUpperCase() + words.slice(1); +}; + +export const fallbackCostLabel = (key: string): string => { + const perUnit = PER_UNIT_KEY.exec(key); + if (perUnit) { + return i18n.t("deploymentPricing.rowLabel", { + charge: perUnit[1] ? sentenceCase(perUnit[1]) : i18n.t("deploymentPricing.fallbackCost"), + unit: perUnit[2].replaceAll("_", " "), + }); + } + const suffixed = COST_SUFFIX.exec(key); + if (suffixed) return sentenceCase([suffixed[1], suffixed[2]].filter(Boolean).join(" ")); + return sentenceCase(key); +}; + +interface ParsedEntry { + readonly key: string; + readonly value: number; + readonly cost: CostKey; +} + +interface PriceEntry extends ParsedEntry { + readonly cost: Price; +} + +const isPriceEntry = (entry: ParsedEntry): entry is PriceEntry => entry.cost.kind === "price"; + +const parseCosts = (costs: Readonly>): readonly ParsedEntry[] => + Object.entries(costs).map(([key, value]) => ({ key, value, cost: parseCostKey(key) })); + +const uniqueBy = (items: readonly T[], keyOf: (item: T) => K): readonly T[] => + items.filter((item, index) => items.findIndex((other) => keyOf(other) === keyOf(item)) === index); + +const byRowRank = (a: PriceEntry, b: PriceEntry): number => compareRanks(rowRank(a.cost), rowRank(b.cost)); + +const groupRows = (entries: readonly PriceEntry[]): readonly PriceRow[] => + [...uniqueBy(entries, ({ cost }) => rowLabel(cost))].sort(byRowRank).map(({ cost }) => ({ + label: rowLabel(cost), + prices: Object.fromEntries( + entries + .filter((other) => rowLabel(other.cost) === rowLabel(cost)) + .map((other) => [other.cost.tier, formatUnitPrice(other.value, other.cost.unit)]), + ), + })); + +export const buildPricingTable = (costs: Readonly>): PricingTable => { + const parsed = parseCosts(costs); + const prices = parsed.filter(isPriceEntry); + const thresholds = [...uniqueBy(prices, ({ cost }) => cost.threshold)] + .map(({ cost }) => cost.threshold) + .sort((a, b) => (a ?? 0) - (b ?? 0)); + const tiers = SERVICE_TIERS.filter((tier) => prices.some(({ cost }) => cost.tier === tier)); + const groups = thresholds.map((threshold) => ({ + threshold, + rows: groupRows(prices.filter(({ cost }) => cost.threshold === threshold)), + })); + return { + tiers, + groups, + hasBlankCells: groups.some(({ rows }) => rows.some(({ prices: row }) => tiers.some((tier) => !(tier in row)))), + unknown: parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => ({ key, label: fallbackCostLabel(key), price: formatRawPrice(value) })), + }; +}; + +export const formatTokenCount = (tokens: number): string => { + if (tokens >= 1_000_000) return `${+(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `${+(tokens / 1_000).toFixed(1)}K`; + return String(tokens); +}; + +const tierPart = ({ cost, value }: PriceEntry): string => { + const charge = chargeLabel(cost, "inline"); + const price = formatUnitPrice(value, cost.unit); + return cost.unit === "token" + ? i18n.t("deploymentPricing.tierPrice", { price, charge }) + : i18n.t("deploymentPricing.tierPricePerUnit", { price, charge, unit: unitLabel(cost.unit) }); +}; + +export const describePricingTier = ({ range: [low, high], costs }: PricingTier): string => { + const parsed = parseCosts(costs); + const parts = [ + ...[...parsed.filter(isPriceEntry)].sort(byRowRank).map(tierPart), + ...parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => + i18n.t("deploymentPricing.tierPrice", { + price: formatRawPrice(value), + charge: fallbackCostLabel(key).toLowerCase(), + }), + ), + ]; + return i18n.t("deploymentPricing.tierRange", { + low: formatTokenCount(low), + high: formatTokenCount(high), + prices: parts.join(" / "), + }); +}; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx index 95901f2bf1d..39495baeea3 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx @@ -4,6 +4,7 @@ import { ChevronDown } from "lucide-react"; import * as React from "react"; import { Trans, useTranslation } from "react-i18next"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; @@ -17,6 +18,7 @@ import { modeLabel, modelCapabilities, providerName, + SECTION_TITLE, wildcardExample, } from "./hubFormat"; import { Badges, CodeBlock, type Fact, Facts, PaneSection, ProviderAvatar, ProviderName } from "./HubParts"; @@ -175,6 +177,10 @@ export function ModelInspector({ model }: { model: ModelGroupInfo }) { )} + {t("publicModelHub.modelInspector.pricingByDeployment")}} + /> {capabilities.length > 0 && ( diff --git a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx index e966836fa3a..cf40a685d76 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx @@ -77,6 +77,13 @@ const MODELS = [ output_cost_per_token: 0, max_input_tokens: 0, description: "General purpose", + deployment_pricing: [ + { + provider: "azure", + costs: { output_cost_per_token: 1e-5, output_cost_per_token_priority: 2e-5 }, + tiered_pricing: [], + }, + ], }, ]; @@ -362,6 +369,24 @@ describe("PublicAiHub", () => { expect(within(pricedRow).queryByRole("cell", { name: "Free" })).not.toBeInTheDocument(); }); + it("shows the open model's prices for each of its deployments", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(await screen.findByRole("link", { name: /gpt-5/ })); + + expect(await screen.findByRole("heading", { name: "Pricing by deployment" })).toBeInTheDocument(); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(azure).getByRole("row", { name: /Output \/ 1M tokens/ })).toHaveTextContent("$10.00$20.00"); + }); + + it("leaves the pricing breakdown out for a model without deployment prices", async () => { + renderWithProviders(, { searchParams: { open: "claude-sonnet-4-5" } }); + + expect(await screen.findByRole("heading", { level: 2, name: "claude-sonnet-4-5" })).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Pricing by deployment" })).not.toBeInTheDocument(); + }); + it("treats an empty open param as nothing opened", async () => { renderWithProviders(, { searchParams: { open: "" } }); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index 515b8230d5b..4081751ca73 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -327,6 +327,33 @@ describe("PublicModelHub", () => { expect(example).toHaveTextContent('model="gpt-4"'); }); + it("shows each deployment's full pricing in the model details", async () => { + respondWith([ + model({ + model_group: "gpt-4", + input_cost_per_token: 2.5e-6, + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, cache_read_input_token_cost_above_272k_tokens_priority: 1e-6 }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8 } }], + }, + ], + }), + ]); + renderHub(); + + fireEvent.click(await screen.findByRole("button", { name: "gpt-4" })); + + const openai = await screen.findByRole("region", { name: "OpenAI pricing" }); + const aboveThreshold = within(openai).getByRole("rowgroup", { name: "Above 272K tokens" }); + expect(within(aboveThreshold).getByRole("row", { name: /Cache read/ })).toHaveTextContent( + "Cache read / 1M tokens$1.00", + ); + expect(within(openai).getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(within(screen.getByRole("dialog")).getByText("$2.5000")).toBeInTheDocument(); + }); + it("filters by several providers at once, and returns to the first page", async () => { const user = userEvent.setup(); respondWith(DEFAULT_MODELS, 300); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index a80bf939165..552ed6fc490 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -56,6 +56,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders"; interface PublicModelHubProps { @@ -954,6 +955,13 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

+ {t("aiHub.publicModelHub.modelDialog.pricingByDeployment")}

+ } + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 919d6cd25cc..3c9dbc99a6b 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -7217,7 +7217,8 @@ "rpm": "Requests per Minute:", "openaiParams": "Supported OpenAI Parameters", "usageExample": "Usage Example", - "copyToClipboard": "Copy to clipboard" + "copyToClipboard": "Copy to clipboard", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7370,7 +7371,8 @@ "usageExample": "Usage Example", "decisionNote": "Decision models answer on /v1/decisions and /v1/systemone, not /chat/completions.", "howToCall": "How to call decision models", - "tryPlayground": "Try it in the Playground" + "tryPlayground": "Try it in the Playground", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7905,7 +7907,8 @@ "title": "Wildcard model", "description": "Call it as , with any model name this provider offers" }, - "capabilities": "Capabilities" + "capabilities": "Capabilities", + "pricingByDeployment": "Pricing by deployment" }, "agentInspector": { "copyName": "Copy agent name", @@ -20300,5 +20303,69 @@ "save": "Save Changes" } } + }, + "deploymentPricing": { + "deploymentCount_one": "This model has {{count}} deployment. Each request is billed at the prices of the deployment that served it.", + "deploymentCount_other": "This model has {{count}} deployments. Each request is billed at the prices of the deployment that served it.", + "requestAdjustments": "These are list prices. Adjustments made per request, like off-peak discounts, regional or data residency surcharges, and some providers' own pricing rules, aren't included.", + "providerPricing": "{{provider}} pricing", + "chargeColumn": "Charge", + "basePrices": "Base prices", + "aboveThreshold": "Above {{threshold}} tokens", + "blankCells": "Blank cells aren't priced separately for this deployment.", + "tieredPricing": "Prices by request size, per 1M tokens", + "tierRange": "{{low}}–{{high}} tokens: {{prices}}", + "tierPrice": "{{price}} {{charge}}", + "tierPricePerUnit": "{{price}} {{charge}} per {{unit}}", + "free": "Free", + "rowLabel": "{{charge}} / {{unit}}", + "withModality": "{{modality}} {{charge}}", + "withCacheDuration": "{{charge}} ({{duration}})", + "fallbackCost": "Cost", + "tiers": { + "standard": "Standard", + "flex": "Flex", + "balanced": "Balanced", + "priority": "Priority", + "ultrafast": "Ultrafast", + "batches": "Batch" + }, + "charges": { + "input": "Input", + "cacheRead": "Cache read", + "explicitCacheRead": "Explicit cache read", + "cacheWrite": "Cache write", + "output": "Output" + }, + "chargesInline": { + "input": "input", + "cacheRead": "cache read", + "explicitCacheRead": "explicit cache read", + "cacheWrite": "cache write", + "output": "output" + }, + "modalities": { + "text": "Text", + "image": "Image", + "audio": "Audio", + "video": "Video", + "reasoning": "Reasoning" + }, + "modalitiesInline": { + "text": "text", + "image": "image", + "audio": "audio", + "video": "video", + "reasoning": "reasoning" + }, + "units": { + "token": "1M tokens", + "second": "second", + "image": "image", + "pixel": "pixel", + "character": "character", + "query": "query", + "request": "request" + } } } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c351e3b1b90..ab1f3c253ea 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32837,6 +32837,20 @@ export interface components { } & { [key: string]: unknown; }; + /** DeploymentPricing */ + DeploymentPricing: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Provider */ + provider: string; + /** + * Tiered Pricing + * @default [] + */ + tiered_pricing: components["schemas"]["PriceTier"][]; + }; /** DeploymentWarning */ DeploymentWarning: { /** Id */ @@ -39687,6 +39701,11 @@ export interface components { } | null; /** Custom Tags */ custom_tags?: string[] | null; + /** + * Deployment Pricing + * @default [] + */ + deployment_pricing: components["schemas"]["DeploymentPricing"][]; /** Description */ description?: string | null; /** Health Checked At */ @@ -42891,6 +42910,18 @@ export interface components { */ version_status: string; }; + /** PriceTier */ + PriceTier: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Range */ + range: [ + number, + number + ]; + }; /** Prompt */ Prompt: { litellm_params: components["schemas"]["PromptLiteLLMParams"];