From 4736c1a2d731197303b240964f24df0c90ba3b28 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 4 Nov 2026 17:49:29 -0800 Subject: [PATCH 1/6] feat(model-hub): show each deployment's full prices in the model hub The model hub only showed one input and one output price per model, the highest across its deployments. Each model group now lists every deployment's prices, read from the same entry billing charges, including cached tokens, long-context tiers, flex, priority, batch and tiered ranges. Both the logged-in and public hub detail views render them in one table per deployment, labeled by provider only --- litellm/router_python_core.py | 12 + litellm/router_utils/model_group_pricing.py | 47 ++++ litellm/types/router.py | 16 ++ .../public_endpoints/test_public_endpoints.py | 51 ++++ tests/unit/test_router/test_router.py | 194 ++++++++++++++ .../components/AIHub/ModelHubTable.test.tsx | 30 ++- .../src/components/AIHub/ModelHubTable.tsx | 6 + .../components/AIHub/ModelHubTableColumns.tsx | 2 + .../components/PublicModelHubTableColumns.tsx | 2 + .../DeploymentPricingBreakdown.test.tsx | 96 +++++++ .../models/DeploymentPricingBreakdown.tsx | 134 ++++++++++ .../models/deploymentPricing.test.ts | 173 +++++++++++++ .../molecules/models/deploymentPricing.ts | 236 ++++++++++++++++++ .../publicModelHub/ModelInspector.tsx | 6 + .../PublicAiHub.integration.test.tsx | 25 ++ .../src/components/public_model_hub.test.tsx | 25 ++ .../src/components/public_model_hub.tsx | 6 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 31 +++ 18 files changed, 1091 insertions(+), 1 deletion(-) create mode 100644 litellm/router_utils/model_group_pricing.py create mode 100644 ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx create mode 100644 ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx create mode 100644 ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts create mode 100644 ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 4b5ecc82581..69ed4319118 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -378,6 +378,7 @@ from litellm.router_utils.kubernetes_pod_discovery import ( resolve_pods_after, resolve_pods_after_bound, ) +from litellm.router_utils.model_group_pricing import deployment_pricing from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, RESPONSES_CONTINUATION_KWARG, @@ -792,6 +793,11 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _configured_deployment_id(deployment: Mapping[str, object]) -> str | None: + deployment_id: Final = _configured_model_info(deployment).get("id") + return deployment_id if isinstance(deployment_id, str) else None + + def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: model_info: Final = _configured_model_info(deployment) catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY) @@ -13526,6 +13532,12 @@ class PythonCore(DeprecatedRouterMembers): if model_info.get("rpm", None) is not None and _deployment_rpm is None: _deployment_rpm = model_info.get("rpm") + if ( + pricing := deployment_pricing( + llm_provider, _configured_deployment_id(model), litellm_params.model, model_info + ) + ) is not None and pricing not in model_group_info.deployment_pricing: + model_group_info.deployment_pricing = (*model_group_info.deployment_pricing, pricing) model_group_info.supported_endpoints = ( tuple( dict.fromkeys( diff --git a/litellm/router_utils/model_group_pricing.py b/litellm/router_utils/model_group_pricing.py new file mode 100644 index 00000000000..b8516071774 --- /dev/null +++ b/litellm/router_utils/model_group_pricing.py @@ -0,0 +1,47 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info +from litellm.types.router import DeploymentPricing, PriceTier + +_RAW_PRICE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def _cost_value(value: object) -> float | None: + match value: + case bool(): + return None + case int() | float(): + return float(value) + case _: + return None + + +def _cost_fields(fields: Mapping[str, object]) -> Mapping[str, float]: + return {name: cost for name, value in fields.items() if "cost" in name and (cost := _cost_value(value)) is not None} + + +def _price_tiers(tiered_pricing: object) -> tuple[PriceTier, ...]: + try: + return tuple( + PriceTier.model_validate({"range": tier.get("range"), "costs": _cost_fields(tier)}) + for tier in _RAW_PRICE_TIERS.validate_python(tiered_pricing) + ) + except ValidationError: + return () + + +def deployment_pricing( + provider: str, deployment_id: str | None, deployment_model: str, resolved_model_info: Mapping[str, object] +) -> DeploymentPricing | None: + """Prices from the entry billing charges this deployment at, so a cost-map variant its own pricing shadows + is never shown. A deployment that declares no pricing is billed at its resolved cost-map entry.""" + billed_model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=deployment_model) + model_info: Final = resolved_model_info if billed_model_info is None else billed_model_info + costs: Final = _cost_fields(model_info) + tiers: Final = _price_tiers(model_info.get("tiered_pricing")) + if not costs and not tiers: + return None + return DeploymentPricing(provider=provider, costs=costs, tiered_pricing=tiers) diff --git a/litellm/types/router.py b/litellm/types/router.py index 4e3e4d92b3c..a727aad2654 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1040,6 +1040,21 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: return MappingProxyType(get_type_hints(model_class)) +class PriceTier(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + range: tuple[float, float] + costs: Mapping[str, float] + + +class DeploymentPricing(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + provider: str + costs: Mapping[str, float] + tiered_pricing: tuple[PriceTier, ...] = () + + class ModelGroupInfo(LiteLLMBaseModel): model_group: str providers: list[str] @@ -1070,6 +1085,7 @@ class ModelGroupInfo(LiteLLMBaseModel): configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None description: str | None = None supported_endpoints: tuple[str, ...] | None = None + deployment_pricing: tuple[DeploymentPricing, ...] = () def __init__(self, **data) -> None: for field_name, field_type in _resolved_annotations(self.__class__).items(): diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index c17a9ba9371..c5cf1da222d 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -985,6 +985,57 @@ def test_public_model_hub_reports_the_newest_check_of_a_group_not_the_last_row_r app.dependency_overrides.clear() +def test_public_model_hub_lists_each_deployments_prices_without_exposing_the_deployment() -> None: + llm_router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-a", + "api_base": "https://secret-a.internal", + "api_key": "sk-secret-a", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"id": "hub-secret-id-a"}, + }, + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-b", + "api_base": "https://secret-b.internal", + "api_key": "sk-secret-b", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, + }, + "model_info": {"id": "hub-secret-id-b"}, + }, + ] + ) + app: Final = FastAPI() + app.include_router(router) + + with ( + patch("litellm.public_model_groups", ["hub-public"]), + patch("litellm.proxy.proxy_server.llm_router", llm_router), + patch("litellm.proxy.proxy_server.prisma_client", None), + ): + response: Final = TestClient(app).get("/public/model_hub") + + assert response.status_code == 200, response.text + groups: Final = TypeAdapter(list[ModelGroupInfoProxy]).validate_json(response.text) + assert [dict(p.costs) for p in groups[0].deployment_pricing] == [ + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}, + {"input_cost_per_token": 3e-06, "output_cost_per_token": 4e-06}, + ] + leaked: Final = [ + secret + for secret in ("secret-a.internal", "secret-b.internal", "sk-secret", "hub-secret-backend", "hub-secret-id") + if secret in response.text + ] + assert leaked == [], response.text + + # --------------------------------------------------------------------------- # /public/agent_hub # --------------------------------------------------------------------------- diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index f5962cb4933..a1ef5562a86 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2534,6 +2534,200 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0 +_HUB_PRICES: Final = MappingProxyType( + { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "cache_creation_input_token_cost": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "output_cost_per_token_above_200k_tokens": 8e-06, + "input_cost_per_token_priority": 3e-06, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_batches": 5e-07, + } +) + + +def _cost_map_model(key: str, entry: Mapping[str, object]) -> str: + litellm.register_model( + {key: {"litellm_provider": "openai", "mode": "chat", **entry}}, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + return f"openai/{key}" + + +def test_model_group_info_lists_every_cost_map_price_of_a_deployment(local_model_cost_map: None) -> None: + model: Final = _cost_map_model("hub-pricing-model", {**_HUB_PRICES, "max_input_tokens": 400000}) + router: Final = litellm.Router(model_list=[{"model_name": "hub", "litellm_params": {"model": model}}]) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [(p.provider, dict(p.costs), p.tiered_pricing) for p in group.deployment_pricing] == [ + ("openai", dict(_HUB_PRICES), ()) + ] + + +def test_model_group_info_lists_each_deployments_price_once_next_to_the_unchanged_max( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + override: Final = 3 * _HUB_PRICES["input_cost_per_token"] + router: Final = litellm.Router( + model_list=[ + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://a.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://b.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "input_cost_per_token": override}}, + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [p.costs["input_cost_per_token"] for p in group.deployment_pricing] == [ + _HUB_PRICES["input_cost_per_token"], + override, + ] + assert group.input_cost_per_token == override + assert group.output_cost_per_token == _HUB_PRICES["output_cost_per_token"] + + +def _usage_billed_at(cost_key: str, prompt_tokens: int = 1000) -> tuple[Usage, str | None]: + match cost_key: + case "input_cost_per_token": + return Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens), None + case "output_cost_per_token": + return Usage(prompt_tokens=0, completion_tokens=1000, total_tokens=1000), None + case "cache_read_input_token_cost": + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details={"cached_tokens": 1000}, + ), None + case "cache_creation_input_token_cost": + return Usage( + prompt_tokens=1000, completion_tokens=0, total_tokens=1000, cache_creation_input_tokens=1000 + ), None + case "input_cost_per_token_priority": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "priority" + case "input_cost_per_token_flex": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "flex" + case _: + raise AssertionError(f"no billing scenario for {cost_key}") + + +def _billed_rate(model: str, deployment_id: str, usage: Usage, service_tier: str | None) -> float: + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + service_tier=service_tier, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + +def test_model_group_info_shows_an_overridden_deployment_only_the_prices_billing_charges( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": { + "model": model, + "input_cost_per_token": 3 * _HUB_PRICES["input_cost_per_token"], + "output_cost_per_token": 3 * _HUB_PRICES["output_cost_per_token"], + }, + "model_info": {"id": "hub-override"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + assert {key: _billed_rate(model, "hub-override", *_usage_billed_at(key)) for key in costs} == pytest.approx( + dict(costs) + ) + long_prompt: Final = _usage_billed_at("input_cost_per_token", prompt_tokens=250_000) + assert _billed_rate(model, "hub-override", *long_prompt) == pytest.approx(costs["input_cost_per_token"]) + shadowed: Final = { + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "input_cost_per_token_batches", + } + assert shadowed.isdisjoint(costs), dict(costs) + + +def test_model_group_info_lists_tiered_pricing_ranges(local_model_cost_map: None) -> None: + tiers: Final = ( + {"range": [0, 256000], "input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}, + {"range": [256000, 1000000], "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}, + ) + model: Final = _cost_map_model("hub-tiered-model", {"tiered_pricing": list(tiers)}) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "tiered", + "litellm_params": {"model": model, "input_cost_per_token": 1e-06, "output_cost_per_token": 4e-06}, + }, + ] + ) + + group: Final = router.registry().group("tiered") + + assert group is not None + assert [ + [(tier.range, dict(tier.costs)) for tier in pricing.tiered_pricing] for pricing in group.deployment_pricing + ] == [ + [ + ((0.0, 256000.0), {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}), + ((256000.0, 1000000.0), {"input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}), + ], + [], + ] + + +def test_model_group_info_lists_custom_prices_of_an_unmapped_deployment_and_skips_an_unpriced_one() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "self-hosted", + "litellm_params": {"model": "openai/hub-unpriced", "api_base": "http://localhost:8000/v1"}, + "model_info": {"cost_tracking_exempt": True}, + }, + { + "model_name": "self-hosted", + "litellm_params": { + "model": "openai/hub-priced", + "api_base": "http://localhost:8001/v1", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"cache_read_input_token_cost": 1e-07}, + }, + ] + ) + + group: Final = router.registry().group("self-hosted") + + assert group is not None + assert [(p.provider, dict(p.costs)) for p in group.deployment_pricing] == [ + ( + "openai", + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06, "cache_read_input_token_cost": 1e-07}, + ) + ] + + def test_model_group_info_description_from_model_info(): """ model_info.description set in the config should surface on ModelGroupInfo, diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 123c7431732..4132f7986c7 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -2,6 +2,7 @@ import * as networking from "@/components/networking"; import userEvent from "@testing-library/user-event"; import { act } from "@testing-library/react"; import type { MCPServerData } from "./MCPHubTableColumns"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import ModelHubTable from "./ModelHubTable"; @@ -374,7 +375,7 @@ describe("ModelHubTable", () => { describe("model details usage example", () => { const openDetails = async ( - model: { model_group: string; providers: string[]; mode: string }, + model: { model_group: string; providers: string[]; mode: string; deployment_pricing?: DeploymentPricing[] }, canOpenPlayground = true, ) => { vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [model] }); @@ -425,6 +426,33 @@ describe("ModelHubTable", () => { expect(screen.queryByRole("link", { name: "Try it in the Playground" })).not.toBeInTheDocument(); }); + it("shows each deployment's own prices, by service tier, in the model details", async () => { + const dialog = await openDetails({ + model_group: "gpt-5", + providers: ["openai", "azure"], + mode: "chat", + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, input_cost_per_token_priority: 5e-6 }, + tiered_pricing: [], + }, + { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }, + ], + }); + + const openai = within(dialog).getByRole("region", { name: "OpenAI pricing" }); + expect(within(openai).getByRole("row", { name: /Input \/ 1M tokens/ })).toHaveTextContent("$2.50$5.00"); + expect(within(dialog).getByRole("region", { name: "Azure pricing" })).toHaveTextContent("$2.75"); + }); + + it("leaves the pricing breakdown out when the proxy sends no deployment prices", async () => { + const dialog = await openDetails({ model_group: "gpt-5", providers: ["openai"], mode: "chat" }); + + expect(within(dialog).getByText("Input Cost per 1M Tokens:")).toBeInTheDocument(); + expect(within(dialog).queryByText("Pricing by Deployment:")).not.toBeInTheDocument(); + }); + it("keeps the chat completions example for a chat model", async () => { const dialog = await openDetails({ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 4d44d0bb809..d60fc072e1a 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -57,6 +57,7 @@ import { isDecisionMode, } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import { ModelCustomMetadata } from "./ModelCustomMetadata"; interface ModelHubTableProps { @@ -819,6 +820,11 @@ const ModelHubTable: React.FC = ({

+ Pricing by Deployment:

} + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 2e4af18455f..7f2f3a49d44 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -22,6 +22,7 @@ import { copyToClipboard } from "@/utils/dataUtils"; import { ModelCustomTags, type ModelCustomMetadataValue } from "./ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "./ModelHubProviders"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelHubData { model_group: string; @@ -41,6 +42,7 @@ export interface ModelHubData { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index 08d6b973b87..eb89af22f17 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -11,6 +11,7 @@ import i18n from "@/i18n/i18n"; import { displayMode } from "@/lib/decisionModels"; import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "@/components/AIHub/ModelHubProviders"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelGroupInfo { model_group: string; @@ -32,6 +33,7 @@ export interface ModelGroupInfo { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx new file mode 100644 index 00000000000..f9f15f3efe3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx @@ -0,0 +1,96 @@ +import { render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { DeploymentPricingBreakdown } from "./DeploymentPricingBreakdown"; +import type { DeploymentPricing } from "./deploymentPricing"; + +const OPENAI: DeploymentPricing = { + provider: "openai", + costs: { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + input_cost_per_token_priority: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + output_cost_per_second_720p: 0.05, + }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8, output_cost_per_token: 4e-7 } }], +}; + +const AZURE: DeploymentPricing = { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }; + +const pricesOf = (region: HTMLElement, group: string, charge: string) => { + const rows = within(within(region).getByRole("rowgroup", { name: group })).getAllByRole("row"); + const row = rows.find((candidate) => within(candidate).queryByRole("rowheader", { name: charge })); + expect(row).toBeDefined(); + return within(row as HTMLElement) + .getAllByRole("cell") + .map((cell) => cell.textContent); +}; + +describe("DeploymentPricingBreakdown", () => { + it("renders one block per deployment, headed by its provider, with that deployment's own prices", () => { + render(Pricing} />); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(openai).getByRole("heading", { name: /OpenAI/ })).toBeInTheDocument(); + expect(pricesOf(openai, "Base prices", "Input / 1M tokens")).toEqual(["$2.50", "$5.00"]); + expect(pricesOf(azure, "Base prices", "Input / 1M tokens")).toEqual(["$2.75"]); + expect(within(azure).queryByText(/Blank cells/)).not.toBeInTheDocument(); + expect(screen.getByText(/This model has 2 deployments/)).toBeInTheDocument(); + }); + + it("lays prices out by service tier column and context threshold group", () => { + render(); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const header = within(openai).getAllByRole("rowgroup")[0]; + expect(within(header).getAllByRole("columnheader").map((cell) => cell.textContent)).toEqual([ + "Charge", + "Standard", + "Priority", + ]); + expect(pricesOf(openai, "Base prices", "Output / 1M tokens")).toEqual(["$10.00", ""]); + expect(pricesOf(openai, "Above 272K tokens", "Cache read / 1M tokens")).toEqual(["", "$1.00"]); + expect(within(openai).getByText("Blank cells aren't priced separately for this deployment.")).toBeInTheDocument(); + expect(screen.queryByText(/This model has/)).not.toBeInTheDocument(); + }); + + it("shows tiered pricing ranges and keys it could not read", () => { + render(); + + expect(screen.getByText("0–256K tokens: $0.05 input / $0.40 output")).toBeInTheDocument(); + expect(screen.getByText("Output / second 720p")).toBeInTheDocument(); + expect(screen.queryByText("output_cost_per_second_720p")).not.toBeInTheDocument(); + expect(screen.getByText("$0.05")).toBeInTheDocument(); + }); + + it("shows only the request-size ranges for a deployment priced purely by tiers", () => { + render( + , + ); + + expect(screen.getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); + + it.each([ + ["missing", undefined], + ["null", null], + ["empty", []], + ])("renders nothing, heading included, when deployment pricing is %s", (_, deployments) => { + const { container } = render( + Pricing by deployment} />, + ); + + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx new file mode 100644 index 00000000000..f1123091696 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx @@ -0,0 +1,134 @@ +import type * as React from "react"; + +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { cn } from "@/lib/cva.config"; +import { + NUMERIC_CELL_CLASS, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +import { + buildPricingTable, + type DeploymentPricing, + describePricingTier, + formatTokenCount, + type PriceGroup, + SERVICE_TIER_LABELS, + type ServiceTier, +} from "./deploymentPricing"; + +const groupName = (threshold: number | null): string => + threshold === null ? "Base prices" : `Above ${formatTokenCount(threshold)} tokens`; + +function PriceGroupRows({ group, tiers }: { group: PriceGroup; tiers: readonly ServiceTier[] }) { + const name = groupName(group.threshold); + return ( + + {group.threshold !== null && ( + + + {name} + + + )} + {group.rows.map((row) => ( + + + {row.label} + + {tiers.map((tier) => ( + + {row.prices[tier]} + + ))} + + ))} + + ); +} + +function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { + const table = buildPricingTable(deployment.costs); + const tiers = deployment.tiered_pricing; + const providerName = getProviderLogoAndName(deployment.provider).displayName; + return ( +
+

+ + {providerName} +

+ {table.groups.length > 0 && ( + + + + Charge + {table.tiers.map((tier) => ( + + {SERVICE_TIER_LABELS[tier]} + + ))} + + + {table.groups.map((group) => ( + + ))} +
+ )} + {table.hasBlankCells && ( +

Blank cells aren't priced separately for this deployment.

+ )} + {tiers.length > 0 && ( +
+

Prices by request size, per 1M tokens

+
    + {tiers.map((tier) => ( +
  • {describePricingTier(tier)}
    • + ))} +
+
+ )} + {table.unknown.length > 0 && ( +
+ {table.unknown.map(({ key, label, price }) => ( +
+
{label}
+
{price}
+
+ ))} +
+ )} +
+ ); +} + +export function DeploymentPricingBreakdown({ + deployments, + heading, + className, +}: { + deployments: readonly DeploymentPricing[] | null | undefined; + heading?: React.ReactNode; + className?: string; +}) { + if (!deployments || deployments.length === 0) return null; + return ( +
+ {heading} + {deployments.length > 1 && ( +

+ This model has {deployments.length} deployments. Each request is billed at the prices of the deployment + that served it. +

+ )} + {deployments.map((deployment, index) => ( + + ))} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts new file mode 100644 index 00000000000..5c1f2c347e2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; + +import { buildPricingTable, describePricingTier, fallbackCostLabel, parseCostKey } from "./deploymentPricing"; + +describe("parseCostKey", () => { + it.each([ + ["input_cost_per_token", { charge: "input", modality: "", unit: "token", threshold: null, tier: "standard" }], + ["output_cost_per_token_flex", { charge: "output", unit: "token", threshold: null, tier: "flex" }], + ["input_cost_per_token_batches", { charge: "input", tier: "batches" }], + ["cache_read_input_token_cost", { charge: "cache_read", unit: "token", tier: "standard" }], + [ + "cache_read_input_token_cost_above_272k_tokens_priority", + { charge: "cache_read", threshold: 272_000, tier: "priority" }, + ], + [ + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", + { charge: "cache_write", cacheDuration: "12hr", threshold: 272_000, tier: "flex" }, + ], + ["cache_creation_input_token_cost_above_1hr", { charge: "cache_write", cacheDuration: "1hr", threshold: null }], + ["cache_read_input_audio_token_cost_priority", { charge: "cache_read", modality: "audio", tier: "priority" }], + ["input_cost_per_audio_token", { charge: "input", modality: "audio", unit: "token" }], + ["output_cost_per_reasoning_token", { charge: "output", modality: "reasoning", unit: "token" }], + [ + "input_cost_per_audio_per_second_above_128k_tokens", + { charge: "input", modality: "audio", unit: "second", threshold: 128_000 }, + ], + ["output_cost_per_second", { charge: "output", modality: "", unit: "second" }], + ["input_cost_per_image", { charge: "input", modality: "", unit: "image" }], + ["input_cost_per_image_token", { charge: "input", modality: "image", unit: "token" }], + ["output_cost_per_pixel", { unit: "pixel" }], + ["input_cost_per_character", { unit: "character" }], + ["input_cost_per_query", { unit: "query" }], + ["input_cost_per_token_above_200k_tokens_ultrafast", { threshold: 200_000, tier: "ultrafast" }], + ["output_cost_per_token_balanced", { tier: "balanced" }], + ])("reads %s", (key, expected) => { + expect(parseCostKey(key)).toMatchObject({ kind: "price", ...expected }); + }); + + it.each([ + "output_cost_per_second_720p", + "output_cost_per_image_1024", + "input_cost_per_token_cache_hit", + "input_cost_per_token_above_272k_tokens_decisions", + "input_cost_per_video_per_second_above_8s_interval", + "ocr_cost_per_page", + "input_dbu_cost_per_token", + "input_cost_per_text_message", + ])("keeps %s as an unknown key rather than guessing", (key) => { + expect(parseCostKey(key)).toEqual({ kind: "unknown", key }); + }); +}); + +const CONTRACT_COSTS = { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + cache_read_input_token_cost: 1.25e-6, + cache_creation_input_token_cost: 3e-6, + input_cost_per_token_priority: 5e-6, + input_cost_per_token_flex: 1.25e-6, + input_cost_per_token_above_200k_tokens: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + input_cost_per_token_batches: 1.25e-6, + output_cost_per_second: 0.001, + input_cost_per_image: 0.002, +}; + +describe("buildPricingTable", () => { + const table = buildPricingTable(CONTRACT_COSTS); + + it("offers a column per service tier the deployment prices, in a fixed order", () => { + expect(table.tiers).toEqual(["standard", "flex", "priority", "batches"]); + }); + + it("groups prices by context threshold, base prices first", () => { + expect(table.groups.map((group) => group.threshold)).toEqual([null, 200_000, 272_000]); + }); + + it("puts every tier of one charge on one row, token prices scaled to 1M tokens", () => { + expect(table.groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "$2.50", flex: "$1.25", priority: "$5.00", batches: "$1.25" } }, + { label: "Input / image", prices: { standard: "$0.002" } }, + { label: "Cache read / 1M tokens", prices: { standard: "$1.25" } }, + { label: "Cache write / 1M tokens", prices: { standard: "$3.00" } }, + { label: "Output / 1M tokens", prices: { standard: "$10.00" } }, + { label: "Output / second", prices: { standard: "$0.001" } }, + ]); + }); + + it("answers a cached read on priority above a threshold from its own group", () => { + expect(table.groups[2].rows).toEqual([{ label: "Cache read / 1M tokens", prices: { priority: "$1.00" } }]); + }); + + it("lists keys it cannot read under their raw name with the unscaled price", () => { + const { groups, unknown } = buildPricingTable({ input_cost_per_token: 1e-6, output_cost_per_second_720p: 0.05 }); + expect(unknown).toEqual([{ key: "output_cost_per_second_720p", label: "Output / second 720p", price: "$0.05" }]); + expect(groups.flatMap((group) => group.rows.map((row) => row.label))).toEqual(["Input / 1M tokens"]); + }); + + it("shows a zero price as free and keeps precision for tiny per-unit prices", () => { + const { groups } = buildPricingTable({ input_cost_per_token: 0, output_cost_per_pixel: 1e-8 }); + expect(groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "Free" } }, + { label: "Output / pixel", prices: { standard: "$0.00000001" } }, + ]); + }); + + it("labels modality and cache duration in the row", () => { + const { groups } = buildPricingTable({ + cache_creation_input_token_cost_above_1hr: 6e-6, + input_cost_per_audio_token: 4e-5, + output_cost_per_reasoning_token: 2e-5, + }); + expect(groups[0].rows.map((row) => row.label)).toEqual([ + "Audio input / 1M tokens", + "Cache write (1hr) / 1M tokens", + "Reasoning output / 1M tokens", + ]); + }); + + it("returns an empty table for a deployment with no prices", () => { + const empty = { tiers: [], groups: [], hasBlankCells: false, unknown: [] }; + expect(buildPricingTable({})).toEqual(empty); + }); + + it("flags blank cells only when some charge lacks a price in a tier the deployment offers", () => { + const everyTierPriced = { + input_cost_per_token: 1e-6, + input_cost_per_token_priority: 2e-6, + output_cost_per_token: 3e-6, + output_cost_per_token_priority: 6e-6, + }; + expect(buildPricingTable(CONTRACT_COSTS).hasBlankCells).toBe(true); + expect(buildPricingTable(everyTierPriced).hasBlankCells).toBe(false); + }); +}); + +describe("fallbackCostLabel", () => { + it.each([ + ["google_maps_grounding_cost_per_query", "Google maps grounding / query"], + ["file_search_cost_per_gb_per_day", "File search / gb per day"], + ["cost_per_second", "Cost / second"], + ["output_cost_per_second_720p", "Output / second 720p"], + ["explicit_cache_read_input_token_cost", "Explicit cache read input token"], + ["some_new_price", "Some new price"], + ])("names %s as %s", (key, label) => { + expect(fallbackCostLabel(key)).toBe(label); + }); +}); + +describe("describePricingTier", () => { + it("states the token range and each price per 1M tokens, input before output", () => { + expect( + describePricingTier({ + range: [0, 256_000], + costs: { output_cost_per_token: 4e-7, input_cost_per_token: 5e-8 }, + }), + ).toBe("0–256K tokens: $0.05 input / $0.40 output"); + }); + + it("names extra charges and non-token units", () => { + expect( + describePricingTier({ + range: [256_000, 1_000_000], + costs: { + input_cost_per_token: 2.5e-7, + output_cost_per_reasoning_token: 1.2e-5, + input_cost_per_query: 0.01, + web_search_cost_per_call: 0.02, + }, + }), + ).toBe("256K–1M tokens: $0.25 input / $0.01 input per query / $12.00 reasoning output / $0.02 web search / call"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts new file mode 100644 index 00000000000..42b92fee565 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts @@ -0,0 +1,236 @@ +import type { components } from "@/lib/http/schema"; + +export type PricingTier = components["schemas"]["PriceTier"]; +export type DeploymentPricing = components["schemas"]["DeploymentPricing"]; + +export const SERVICE_TIERS = ["standard", "flex", "balanced", "priority", "ultrafast", "batches"] as const; +export type ServiceTier = (typeof SERVICE_TIERS)[number]; + +export const SERVICE_TIER_LABELS: Readonly> = { + standard: "Standard", + flex: "Flex", + balanced: "Balanced", + priority: "Priority", + ultrafast: "Ultrafast", + batches: "Batch", +}; + +const UNITS = ["token", "second", "image", "pixel", "character", "query", "request"] as const; +type Unit = (typeof UNITS)[number]; + +const CHARGES = ["input", "cache_read", "cache_write", "output"] as const; +type Charge = (typeof CHARGES)[number]; + +const MODALITIES = ["", "text", "image", "audio", "video", "reasoning"] as const; +type Modality = (typeof MODALITIES)[number]; + +export type CostKey = + | { + readonly kind: "price"; + readonly charge: Charge; + readonly modality: Modality; + readonly cacheDuration: string; + readonly unit: Unit; + readonly threshold: number | null; + readonly tier: ServiceTier; + } + | { readonly kind: "unknown"; readonly key: string }; + +const TIER_SUFFIX = /^(.+)_(flex|balanced|priority|ultrafast|batches)$/; +const THRESHOLD_SUFFIX = /^(.+)_above_(\d+)k_tokens$/; +const CACHE_DURATION_SUFFIX = /^(.+)_above_(\d+hr)$/; +const DIRECT_PRICE = /^(input|output)_cost_per_(?:(text|image|audio|video|reasoning)_(?:per_)?)?([a-z]+)$/; +const CACHE_PRICE = /^cache_(read|creation)_input_(?:(image|audio)_)?token_cost$/; + +const splitSuffix = (key: string, pattern: RegExp): readonly [string, string | null] => { + const match = pattern.exec(key); + return match ? [match[1], match[2]] : [key, null]; +}; + +const isUnit = (value: string): value is Unit => (UNITS as readonly string[]).includes(value); + +const parseBase = (base: string): Pick, "charge" | "modality" | "unit"> | null => { + const direct = DIRECT_PRICE.exec(base); + if (direct) { + const [, charge, modality = "", unit] = direct; + return isUnit(unit) ? { charge: charge as Charge, modality: modality as Modality, unit } : null; + } + const cache = CACHE_PRICE.exec(base); + if (cache) { + const [, kind, modality = ""] = cache; + return { charge: kind === "read" ? "cache_read" : "cache_write", modality: modality as Modality, unit: "token" }; + } + return null; +}; + +export const parseCostKey = (key: string): CostKey => { + const [withoutTier, tier] = splitSuffix(key, TIER_SUFFIX); + const [withoutThreshold, thousands] = splitSuffix(withoutTier, THRESHOLD_SUFFIX); + const [base, cacheDuration] = splitSuffix(withoutThreshold, CACHE_DURATION_SUFFIX); + const parsed = parseBase(base); + if (!parsed) return { kind: "unknown", key }; + return { + kind: "price", + ...parsed, + cacheDuration: cacheDuration ?? "", + threshold: thousands === null ? null : Number(thousands) * 1000, + tier: (tier ?? "standard") as ServiceTier, + }; +}; + +type Price = Extract; + +const CHARGE_LABELS: Readonly> = { + input: "Input", + cache_read: "Cache read", + cache_write: "Cache write", + output: "Output", +}; + +const chargeLabel = ({ charge, modality, cacheDuration }: Price): string => { + const name = modality ? `${modality} ${CHARGE_LABELS[charge].toLowerCase()}` : CHARGE_LABELS[charge]; + const capitalized = name.charAt(0).toUpperCase() + name.slice(1); + return cacheDuration ? `${capitalized} (${cacheDuration})` : capitalized; +}; + +const unitLabel = (unit: Unit): string => (unit === "token" ? "1M tokens" : unit); + +const rowLabel = (price: Price): string => `${chargeLabel(price)} / ${unitLabel(price.unit)}`; + +const rowRank = (price: Price): readonly number[] => [ + CHARGES.indexOf(price.charge), + UNITS.indexOf(price.unit), + MODALITIES.indexOf(price.modality), + parseInt(price.cacheDuration || "0", 10), +]; + +const compareRanks = (a: readonly number[], b: readonly number[]): number => + a.reduce((result, value, index) => result || value - b[index], 0); + +const PRICE_FORMAT_OPTIONS: Intl.NumberFormatOptions = { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 10, + useGrouping: false, +}; + +const PRICE_FORMAT = new Intl.NumberFormat("en-US", PRICE_FORMAT_OPTIONS); + +export const formatRawPrice = (value: number): string => (value === 0 ? "Free" : PRICE_FORMAT.format(value)); + +const formatUnitPrice = (value: number, unit: Unit): string => + formatRawPrice(unit === "token" ? value * 1_000_000 : value); + +export interface PriceRow { + readonly label: string; + readonly prices: Readonly>>; +} + +export interface PriceGroup { + readonly threshold: number | null; + readonly rows: readonly PriceRow[]; +} + +export interface UnknownPrice { + readonly key: string; + readonly label: string; + readonly price: string; +} + +export interface PricingTable { + readonly tiers: readonly ServiceTier[]; + readonly groups: readonly PriceGroup[]; + readonly hasBlankCells: boolean; + readonly unknown: readonly UnknownPrice[]; +} + +const PER_UNIT_KEY = /^(.*?)_?cost_per_(.+)$/; +const COST_SUFFIX = /^(.+?)_cost(?:_(.+))?$/; + +const sentenceCase = (text: string): string => { + const words = text.replaceAll("_", " ").trim(); + return words.charAt(0).toUpperCase() + words.slice(1); +}; + +export const fallbackCostLabel = (key: string): string => { + const perUnit = PER_UNIT_KEY.exec(key); + if (perUnit) return `${sentenceCase(perUnit[1] || "cost")} / ${perUnit[2].replaceAll("_", " ")}`; + const suffixed = COST_SUFFIX.exec(key); + if (suffixed) return sentenceCase([suffixed[1], suffixed[2]].filter(Boolean).join(" ")); + return sentenceCase(key); +}; + +interface ParsedEntry { + readonly key: string; + readonly value: number; + readonly cost: CostKey; +} + +interface PriceEntry extends ParsedEntry { + readonly cost: Price; +} + +const isPriceEntry = (entry: ParsedEntry): entry is PriceEntry => entry.cost.kind === "price"; + +const parseCosts = (costs: Readonly>): readonly ParsedEntry[] => + Object.entries(costs).map(([key, value]) => ({ key, value, cost: parseCostKey(key) })); + +const uniqueBy = (items: readonly T[], keyOf: (item: T) => K): readonly T[] => + items.filter((item, index) => items.findIndex((other) => keyOf(other) === keyOf(item)) === index); + +const byRowRank = (a: PriceEntry, b: PriceEntry): number => compareRanks(rowRank(a.cost), rowRank(b.cost)); + +const groupRows = (entries: readonly PriceEntry[]): readonly PriceRow[] => + [...uniqueBy(entries, ({ cost }) => rowLabel(cost))].sort(byRowRank).map(({ cost }) => ({ + label: rowLabel(cost), + prices: Object.fromEntries( + entries + .filter((other) => rowLabel(other.cost) === rowLabel(cost)) + .map((other) => [other.cost.tier, formatUnitPrice(other.value, other.cost.unit)]), + ), + })); + +export const buildPricingTable = (costs: Readonly>): PricingTable => { + const parsed = parseCosts(costs); + const prices = parsed.filter(isPriceEntry); + const thresholds = [...uniqueBy(prices, ({ cost }) => cost.threshold)] + .map(({ cost }) => cost.threshold) + .sort((a, b) => (a ?? 0) - (b ?? 0)); + const tiers = SERVICE_TIERS.filter((tier) => prices.some(({ cost }) => cost.tier === tier)); + const groups = thresholds.map((threshold) => ({ + threshold, + rows: groupRows(prices.filter(({ cost }) => cost.threshold === threshold)), + })); + return { + tiers, + groups, + hasBlankCells: groups.some(({ rows }) => rows.some(({ prices: row }) => tiers.some((tier) => !(tier in row)))), + unknown: parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => ({ key, label: fallbackCostLabel(key), price: formatRawPrice(value) })), + }; +}; + +export const formatTokenCount = (tokens: number): string => { + if (tokens >= 1_000_000) return `${+(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `${+(tokens / 1_000).toFixed(1)}K`; + return String(tokens); +}; + +const tierPart = ({ cost, value }: PriceEntry): string => { + const charge = chargeLabel(cost).toLowerCase(); + const price = formatUnitPrice(value, cost.unit); + return cost.unit === "token" ? `${price} ${charge}` : `${price} ${charge} per ${cost.unit}`; +}; + +export const describePricingTier = ({ range: [low, high], costs }: PricingTier): string => { + const parsed = parseCosts(costs); + const parts = [ + ...[...parsed.filter(isPriceEntry)].sort(byRowRank).map(tierPart), + ...parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => `${formatRawPrice(value)} ${fallbackCostLabel(key).toLowerCase()}`), + ]; + return `${formatTokenCount(low)}–${formatTokenCount(high)} tokens: ${parts.join(" / ")}`; +}; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx index 95901f2bf1d..5f6cd621d76 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx @@ -4,6 +4,7 @@ import { ChevronDown } from "lucide-react"; import * as React from "react"; import { Trans, useTranslation } from "react-i18next"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; @@ -17,6 +18,7 @@ import { modeLabel, modelCapabilities, providerName, + SECTION_TITLE, wildcardExample, } from "./hubFormat"; import { Badges, CodeBlock, type Fact, Facts, PaneSection, ProviderAvatar, ProviderName } from "./HubParts"; @@ -175,6 +177,10 @@ export function ModelInspector({ model }: { model: ModelGroupInfo }) { )} + Pricing by deployment} + /> {capabilities.length > 0 && ( diff --git a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx index bcddc1fe364..03c8c8fd595 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx @@ -62,6 +62,13 @@ const MODELS = [ input_cost_per_token: 0, output_cost_per_token: 0, description: "General purpose", + deployment_pricing: [ + { + provider: "azure", + costs: { output_cost_per_token: 1e-5, output_cost_per_token_priority: 2e-5 }, + tiered_pricing: [], + }, + ], }, ]; @@ -272,6 +279,24 @@ describe("PublicAiHub", () => { expect(within(row).getAllByText("Free").length).toBeGreaterThan(0); }); + it("shows the open model's prices for each of its deployments", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(await screen.findByRole("link", { name: /gpt-5/ })); + + expect(await screen.findByRole("heading", { name: "Pricing by deployment" })).toBeInTheDocument(); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(azure).getByRole("row", { name: /Output \/ 1M tokens/ })).toHaveTextContent("$10.00$20.00"); + }); + + it("leaves the pricing breakdown out for a model without deployment prices", async () => { + renderWithProviders(); + + expect(await screen.findByRole("heading", { level: 2, name: "claude-sonnet-4-5" })).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Pricing by deployment" })).not.toBeInTheDocument(); + }); + it("treats an empty open param as nothing opened", async () => { renderWithProviders(, { searchParams: { open: "" } }); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index 515b8230d5b..ed58cc244e7 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -327,6 +327,31 @@ describe("PublicModelHub", () => { expect(example).toHaveTextContent('model="gpt-4"'); }); + it("shows each deployment's full pricing in the model details", async () => { + respondWith([ + model({ + model_group: "gpt-4", + input_cost_per_token: 2.5e-6, + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, cache_read_input_token_cost_above_272k_tokens_priority: 1e-6 }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8 } }], + }, + ], + }), + ]); + renderHub(); + + fireEvent.click(await screen.findByRole("button", { name: "gpt-4" })); + + const openai = await screen.findByRole("region", { name: "OpenAI pricing" }); + const aboveThreshold = within(openai).getByRole("rowgroup", { name: "Above 272K tokens" }); + expect(within(aboveThreshold).getByRole("row", { name: /Cache read/ })).toHaveTextContent("Cache read / 1M tokens$1.00"); + expect(within(openai).getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(within(screen.getByRole("dialog")).getByText("$2.5000")).toBeInTheDocument(); + }); + it("filters by several providers at once, and returns to the first page", async () => { const user = userEvent.setup(); respondWith(DEFAULT_MODELS, 300); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index a80bf939165..cde45c3d551 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -56,6 +56,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders"; interface PublicModelHubProps { @@ -954,6 +955,11 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

+ Pricing by Deployment:

} + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e65a30f6e1d..2a2df66e235 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32335,6 +32335,20 @@ export interface components { } & { [key: string]: unknown; }; + /** DeploymentPricing */ + DeploymentPricing: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Provider */ + provider: string; + /** + * Tiered Pricing + * @default [] + */ + tiered_pricing: components["schemas"]["PriceTier"][]; + }; /** DeploymentWarning */ DeploymentWarning: { /** Id */ @@ -39089,6 +39103,11 @@ export interface components { } | null; /** Custom Tags */ custom_tags?: string[] | null; + /** + * Deployment Pricing + * @default [] + */ + deployment_pricing: components["schemas"]["DeploymentPricing"][]; /** Description */ description?: string | null; /** Health Checked At */ @@ -42243,6 +42262,18 @@ export interface components { */ version_status: string; }; + /** PriceTier */ + PriceTier: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Range */ + range: [ + number, + number + ]; + }; /** Prompt */ Prompt: { litellm_params: components["schemas"]["PromptLiteLLMParams"]; From 34670d3acb96eae84545f1fbc456ab256f996fb5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 4 Nov 2026 18:00:08 -0800 Subject: [PATCH 2/6] refactor(ui): move model hub pricing breakdown text onto i18n Route every string the deployment pricing breakdown writes through t() under a new deploymentPricing group in en.json, plus the three hub headings that host it. The English on screen is unchanged --- .../src/components/AIHub/ModelHubTable.tsx | 2 +- .../models/DeploymentPricingBreakdown.tsx | 30 +++-- .../molecules/models/deploymentPricing.ts | 113 ++++++++++++++---- .../publicModelHub/ModelInspector.tsx | 2 +- .../src/components/public_model_hub.tsx | 4 +- ui/litellm-dashboard/src/i18n/locales/en.json | 70 ++++++++++- 6 files changed, 178 insertions(+), 43 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index d60fc072e1a..60259b1f7f0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -823,7 +823,7 @@ const ModelHubTable: React.FC = ({ Pricing by Deployment:

} + heading={

{t("aiHub.modelHubTable.modelDialog.pricingByDeployment")}

} /> diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx index f1123091696..96f047e9d1b 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx @@ -1,4 +1,5 @@ import type * as React from "react"; +import { useTranslation } from "react-i18next"; import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; @@ -23,11 +24,12 @@ import { type ServiceTier, } from "./deploymentPricing"; -const groupName = (threshold: number | null): string => - threshold === null ? "Base prices" : `Above ${formatTokenCount(threshold)} tokens`; - function PriceGroupRows({ group, tiers }: { group: PriceGroup; tiers: readonly ServiceTier[] }) { - const name = groupName(group.threshold); + const { t } = useTranslation(); + const name = + group.threshold === null + ? t("deploymentPricing.basePrices") + : t("deploymentPricing.aboveThreshold", { threshold: formatTokenCount(group.threshold) }); return ( {group.threshold !== null && ( @@ -54,11 +56,15 @@ function PriceGroupRows({ group, tiers }: { group: PriceGroup; tiers: readonly S } function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { + const { t } = useTranslation(); const table = buildPricingTable(deployment.costs); const tiers = deployment.tiered_pricing; const providerName = getProviderLogoAndName(deployment.provider).displayName; return ( -
+

{providerName} @@ -67,10 +73,10 @@ function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { - Charge + {t("deploymentPricing.chargeColumn")} {table.tiers.map((tier) => ( - {SERVICE_TIER_LABELS[tier]} + {t(SERVICE_TIER_LABELS[tier])} ))} @@ -80,12 +86,10 @@ function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { ))}
)} - {table.hasBlankCells && ( -

Blank cells aren't priced separately for this deployment.

- )} + {table.hasBlankCells &&

{t("deploymentPricing.blankCells")}

} {tiers.length > 0 && (
-

Prices by request size, per 1M tokens

+

{t("deploymentPricing.tieredPricing")}

    {tiers.map((tier) => (
  • {describePricingTier(tier)}
    • @@ -116,14 +120,14 @@ export function DeploymentPricingBreakdown({ heading?: React.ReactNode; className?: string; }) { + const { t } = useTranslation(); if (!deployments || deployments.length === 0) return null; return (
    {heading} {deployments.length > 1 && (

    - This model has {deployments.length} deployments. Each request is billed at the prices of the deployment - that served it. + {t("deploymentPricing.deploymentCount", { count: deployments.length })}

    )} {deployments.map((deployment, index) => ( diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts index 42b92fee565..940dc198f2b 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts @@ -1,3 +1,6 @@ +import type { ParseKeys } from "i18next"; + +import i18n from "@/i18n/i18n"; import type { components } from "@/lib/http/schema"; export type PricingTier = components["schemas"]["PriceTier"]; @@ -6,13 +9,13 @@ export type DeploymentPricing = components["schemas"]["DeploymentPricing"]; export const SERVICE_TIERS = ["standard", "flex", "balanced", "priority", "ultrafast", "batches"] as const; export type ServiceTier = (typeof SERVICE_TIERS)[number]; -export const SERVICE_TIER_LABELS: Readonly> = { - standard: "Standard", - flex: "Flex", - balanced: "Balanced", - priority: "Priority", - ultrafast: "Ultrafast", - batches: "Batch", +export const SERVICE_TIER_LABELS: Readonly> = { + standard: "deploymentPricing.tiers.standard", + flex: "deploymentPricing.tiers.flex", + balanced: "deploymentPricing.tiers.balanced", + priority: "deploymentPricing.tiers.priority", + ultrafast: "deploymentPricing.tiers.ultrafast", + batches: "deploymentPricing.tiers.batches", }; const UNITS = ["token", "second", "image", "pixel", "character", "query", "request"] as const; @@ -80,22 +83,67 @@ export const parseCostKey = (key: string): CostKey => { type Price = Extract; -const CHARGE_LABELS: Readonly> = { - input: "Input", - cache_read: "Cache read", - cache_write: "Cache write", - output: "Output", +type LabelForm = "start" | "inline"; + +const CHARGE_LABELS: Readonly>>> = { + start: { + input: "deploymentPricing.charges.input", + cache_read: "deploymentPricing.charges.cacheRead", + cache_write: "deploymentPricing.charges.cacheWrite", + output: "deploymentPricing.charges.output", + }, + inline: { + input: "deploymentPricing.chargesInline.input", + cache_read: "deploymentPricing.chargesInline.cacheRead", + cache_write: "deploymentPricing.chargesInline.cacheWrite", + output: "deploymentPricing.chargesInline.output", + }, }; -const chargeLabel = ({ charge, modality, cacheDuration }: Price): string => { - const name = modality ? `${modality} ${CHARGE_LABELS[charge].toLowerCase()}` : CHARGE_LABELS[charge]; - const capitalized = name.charAt(0).toUpperCase() + name.slice(1); - return cacheDuration ? `${capitalized} (${cacheDuration})` : capitalized; +const MODALITY_LABELS: Readonly, ParseKeys>>>> = { + start: { + text: "deploymentPricing.modalities.text", + image: "deploymentPricing.modalities.image", + audio: "deploymentPricing.modalities.audio", + video: "deploymentPricing.modalities.video", + reasoning: "deploymentPricing.modalities.reasoning", + }, + inline: { + text: "deploymentPricing.modalitiesInline.text", + image: "deploymentPricing.modalitiesInline.image", + audio: "deploymentPricing.modalitiesInline.audio", + video: "deploymentPricing.modalitiesInline.video", + reasoning: "deploymentPricing.modalitiesInline.reasoning", + }, }; -const unitLabel = (unit: Unit): string => (unit === "token" ? "1M tokens" : unit); +const UNIT_LABELS: Readonly> = { + token: "deploymentPricing.units.token", + second: "deploymentPricing.units.second", + image: "deploymentPricing.units.image", + pixel: "deploymentPricing.units.pixel", + character: "deploymentPricing.units.character", + query: "deploymentPricing.units.query", + request: "deploymentPricing.units.request", +}; -const rowLabel = (price: Price): string => `${chargeLabel(price)} / ${unitLabel(price.unit)}`; +const chargeName = ({ charge, modality }: Price, form: LabelForm): string => + modality + ? i18n.t("deploymentPricing.withModality", { + modality: i18n.t(MODALITY_LABELS[form][modality]), + charge: i18n.t(CHARGE_LABELS.inline[charge]), + }) + : i18n.t(CHARGE_LABELS[form][charge]); + +const chargeLabel = (price: Price, form: LabelForm): string => + price.cacheDuration + ? i18n.t("deploymentPricing.withCacheDuration", { charge: chargeName(price, form), duration: price.cacheDuration }) + : chargeName(price, form); + +const unitLabel = (unit: Unit): string => i18n.t(UNIT_LABELS[unit]); + +const rowLabel = (price: Price): string => + i18n.t("deploymentPricing.rowLabel", { charge: chargeLabel(price, "start"), unit: unitLabel(price.unit) }); const rowRank = (price: Price): readonly number[] => [ CHARGES.indexOf(price.charge), @@ -117,7 +165,8 @@ const PRICE_FORMAT_OPTIONS: Intl.NumberFormatOptions = { const PRICE_FORMAT = new Intl.NumberFormat("en-US", PRICE_FORMAT_OPTIONS); -export const formatRawPrice = (value: number): string => (value === 0 ? "Free" : PRICE_FORMAT.format(value)); +export const formatRawPrice = (value: number): string => + value === 0 ? i18n.t("deploymentPricing.free") : PRICE_FORMAT.format(value); const formatUnitPrice = (value: number, unit: Unit): string => formatRawPrice(unit === "token" ? value * 1_000_000 : value); @@ -155,7 +204,12 @@ const sentenceCase = (text: string): string => { export const fallbackCostLabel = (key: string): string => { const perUnit = PER_UNIT_KEY.exec(key); - if (perUnit) return `${sentenceCase(perUnit[1] || "cost")} / ${perUnit[2].replaceAll("_", " ")}`; + if (perUnit) { + return i18n.t("deploymentPricing.rowLabel", { + charge: perUnit[1] ? sentenceCase(perUnit[1]) : i18n.t("deploymentPricing.fallbackCost"), + unit: perUnit[2].replaceAll("_", " "), + }); + } const suffixed = COST_SUFFIX.exec(key); if (suffixed) return sentenceCase([suffixed[1], suffixed[2]].filter(Boolean).join(" ")); return sentenceCase(key); @@ -219,9 +273,11 @@ export const formatTokenCount = (tokens: number): string => { }; const tierPart = ({ cost, value }: PriceEntry): string => { - const charge = chargeLabel(cost).toLowerCase(); + const charge = chargeLabel(cost, "inline"); const price = formatUnitPrice(value, cost.unit); - return cost.unit === "token" ? `${price} ${charge}` : `${price} ${charge} per ${cost.unit}`; + return cost.unit === "token" + ? i18n.t("deploymentPricing.tierPrice", { price, charge }) + : i18n.t("deploymentPricing.tierPricePerUnit", { price, charge, unit: unitLabel(cost.unit) }); }; export const describePricingTier = ({ range: [low, high], costs }: PricingTier): string => { @@ -230,7 +286,16 @@ export const describePricingTier = ({ range: [low, high], costs }: PricingTier): ...[...parsed.filter(isPriceEntry)].sort(byRowRank).map(tierPart), ...parsed .filter((entry) => !isPriceEntry(entry)) - .map(({ key, value }) => `${formatRawPrice(value)} ${fallbackCostLabel(key).toLowerCase()}`), + .map(({ key, value }) => + i18n.t("deploymentPricing.tierPrice", { + price: formatRawPrice(value), + charge: fallbackCostLabel(key).toLowerCase(), + }), + ), ]; - return `${formatTokenCount(low)}–${formatTokenCount(high)} tokens: ${parts.join(" / ")}`; + return i18n.t("deploymentPricing.tierRange", { + low: formatTokenCount(low), + high: formatTokenCount(high), + prices: parts.join(" / "), + }); }; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx index 5f6cd621d76..39495baeea3 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx @@ -179,7 +179,7 @@ export function ModelInspector({ model }: { model: ModelGroupInfo }) { Pricing by deployment

} + heading={

{t("publicModelHub.modelInspector.pricingByDeployment")}

} /> {capabilities.length > 0 && ( diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index cde45c3d551..552ed6fc490 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -958,7 +958,9 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded Pricing by Deployment:

} + heading={ +

{t("aiHub.publicModelHub.modelDialog.pricingByDeployment")}

+ } /> diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index d3eaf3037b3..bfad499016f 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -6514,7 +6514,8 @@ "rpm": "Requests per Minute:", "openaiParams": "Supported OpenAI Parameters", "usageExample": "Usage Example", - "copyToClipboard": "Copy to clipboard" + "copyToClipboard": "Copy to clipboard", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -6667,7 +6668,8 @@ "usageExample": "Usage Example", "decisionNote": "Decision models answer on /v1/decisions and /v1/systemone, not /chat/completions.", "howToCall": "How to call decision models", - "tryPlayground": "Try it in the Playground" + "tryPlayground": "Try it in the Playground", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7156,7 +7158,8 @@ "title": "Wildcard model", "description": "Call it as , with any model name this provider offers" }, - "capabilities": "Capabilities" + "capabilities": "Capabilities", + "pricingByDeployment": "Pricing by deployment" }, "agentInspector": { "copyName": "Copy agent name", @@ -18465,5 +18468,66 @@ "save": "Save Changes" } } + }, + "deploymentPricing": { + "deploymentCount_one": "This model has {{count}} deployment. Each request is billed at the prices of the deployment that served it.", + "deploymentCount_other": "This model has {{count}} deployments. Each request is billed at the prices of the deployment that served it.", + "providerPricing": "{{provider}} pricing", + "chargeColumn": "Charge", + "basePrices": "Base prices", + "aboveThreshold": "Above {{threshold}} tokens", + "blankCells": "Blank cells aren't priced separately for this deployment.", + "tieredPricing": "Prices by request size, per 1M tokens", + "tierRange": "{{low}}–{{high}} tokens: {{prices}}", + "tierPrice": "{{price}} {{charge}}", + "tierPricePerUnit": "{{price}} {{charge}} per {{unit}}", + "free": "Free", + "rowLabel": "{{charge}} / {{unit}}", + "withModality": "{{modality}} {{charge}}", + "withCacheDuration": "{{charge}} ({{duration}})", + "fallbackCost": "Cost", + "tiers": { + "standard": "Standard", + "flex": "Flex", + "balanced": "Balanced", + "priority": "Priority", + "ultrafast": "Ultrafast", + "batches": "Batch" + }, + "charges": { + "input": "Input", + "cacheRead": "Cache read", + "cacheWrite": "Cache write", + "output": "Output" + }, + "chargesInline": { + "input": "input", + "cacheRead": "cache read", + "cacheWrite": "cache write", + "output": "output" + }, + "modalities": { + "text": "Text", + "image": "Image", + "audio": "Audio", + "video": "Video", + "reasoning": "Reasoning" + }, + "modalitiesInline": { + "text": "text", + "image": "image", + "audio": "audio", + "video": "video", + "reasoning": "reasoning" + }, + "units": { + "token": "1M tokens", + "second": "second", + "image": "image", + "pixel": "pixel", + "character": "character", + "query": "query", + "request": "request" + } } } From e2e5b0e1b9f32e26389fad05b831a973a749ab66 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 5 Nov 2026 19:05:01 -0800 Subject: [PATCH 3/6] fix(model-hub): show Vertex regional prices and explicit cached reads as billed A Vertex deployment outside the global endpoint now shows its token prices with the regional uplift billing applies at the deployment's location. The explicit cached read price shows per 1M tokens on its own row instead of a raw per-token value. The breakdown also says that adjustments made per request, like off-peak discounts and regional or data residency surcharges, aren't included --- .../litellm_core_utils/llm_cost_calc/utils.py | 2 +- litellm/router_rust/surface/model_info.py | 4 ++ litellm/router_utils/model_group_pricing.py | 27 +++++++--- tests/unit/test_router/test_router.py | 49 +++++++++++++++++++ .../DeploymentPricingBreakdown.test.tsx | 11 +++-- .../models/DeploymentPricingBreakdown.tsx | 1 + .../models/deploymentPricing.test.ts | 13 +++++ .../molecules/models/deploymentPricing.ts | 12 +++-- ui/litellm-dashboard/src/i18n/locales/en.json | 3 ++ 9 files changed, 104 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9ac4f9d5ee8..5eb9cdb11ce 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1464,7 +1464,7 @@ def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | N return 1.0 -def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: +def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float: """ Resolve the per-model uplift multiplier for Vertex AI non-global (regional and multi-region) endpoints. diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index b8352a0a926..5d20a99737e 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -46,6 +46,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( from litellm.litellm_core_utils.batch_utils import deployment_supported_endpoints from litellm.litellm_core_utils.core_helpers import as_str_mapping, coerce_token_limit from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.litellm_logging import resolve_vertex_location_for_cost from litellm.litellm_core_utils.reasoning_effort_capability import ( deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -344,6 +345,9 @@ class HostModelInfo: _configured_deployment_id(deployment), group.params[int(str(step["index"]))].model, model_info, + resolve_vertex_location_for_cost( + llm_provider, deployment.get("litellm_params"), None, litellm_model + ), ) if pricing is not None and pricing not in group.pricing: group.pricing = (*group.pricing, pricing) # rebind-ok: the merge's state diff --git a/litellm/router_utils/model_group_pricing.py b/litellm/router_utils/model_group_pricing.py index b8516071774..b4ee63c3ef4 100644 --- a/litellm/router_utils/model_group_pricing.py +++ b/litellm/router_utils/model_group_pricing.py @@ -4,6 +4,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import get_vertex_regional_endpoint_uplift from litellm.types.router import DeploymentPricing, PriceTier _RAW_PRICE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -19,14 +20,18 @@ def _cost_value(value: object) -> float | None: return None -def _cost_fields(fields: Mapping[str, object]) -> Mapping[str, float]: - return {name: cost for name, value in fields.items() if "cost" in name and (cost := _cost_value(value)) is not None} +def _cost_fields(fields: Mapping[str, object], token_uplift: float) -> Mapping[str, float]: + return { + name: cost * token_uplift if "token" in name else cost + for name, value in fields.items() + if "cost" in name and (cost := _cost_value(value)) is not None + } -def _price_tiers(tiered_pricing: object) -> tuple[PriceTier, ...]: +def _price_tiers(tiered_pricing: object, token_uplift: float) -> tuple[PriceTier, ...]: try: return tuple( - PriceTier.model_validate({"range": tier.get("range"), "costs": _cost_fields(tier)}) + PriceTier.model_validate({"range": tier.get("range"), "costs": _cost_fields(tier, token_uplift)}) for tier in _RAW_PRICE_TIERS.validate_python(tiered_pricing) ) except ValidationError: @@ -34,14 +39,20 @@ def _price_tiers(tiered_pricing: object) -> tuple[PriceTier, ...]: def deployment_pricing( - provider: str, deployment_id: str | None, deployment_model: str, resolved_model_info: Mapping[str, object] + provider: str, + deployment_id: str | None, + deployment_model: str, + resolved_model_info: Mapping[str, object], + vertex_location: str | None, ) -> DeploymentPricing | None: """Prices from the entry billing charges this deployment at, so a cost-map variant its own pricing shadows - is never shown. A deployment that declares no pricing is billed at its resolved cost-map entry.""" + is never shown. A deployment that declares no pricing is billed at its resolved cost-map entry. Token prices + carry the Vertex regional-endpoint uplift billing applies at the deployment's location.""" billed_model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=deployment_model) model_info: Final = resolved_model_info if billed_model_info is None else billed_model_info - costs: Final = _cost_fields(model_info) - tiers: Final = _price_tiers(model_info.get("tiered_pricing")) + token_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + costs: Final = _cost_fields(model_info, token_uplift) + tiers: Final = _price_tiers(model_info.get("tiered_pricing"), token_uplift) if not costs and not tiers: return None return DeploymentPricing(provider=provider, costs=costs, tiered_pricing=tiers) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a3ddb2ed2a8..5470f2bbf06 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2029,6 +2029,55 @@ def test_model_group_info_shows_an_overridden_deployment_only_the_prices_billing assert shadowed.isdisjoint(costs), dict(costs) +@pytest.mark.parametrize("vertex_location", ["us-east5", "global"]) +def test_model_group_info_shows_a_vertex_deployment_the_token_prices_billing_charges_at_its_location( + local_model_cost_map: None, vertex_location: str +) -> None: + model: Final = "vertex_ai/hub-vertex-model" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "regional_endpoint_uplift_multiplier": 1.1, + } + }, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": vertex_location}, + "model_info": {"id": "hub-vertex"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + + def billed(cost_key: str) -> float: + usage, _ = _usage_billed_at(cost_key) + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="vertex_ai", + router_model_id="hub-vertex", + vertex_location=vertex_location, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + shown: Final = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") + assert {key: costs[key] for key in shown} == pytest.approx({key: billed(key) for key in shown}) + + def test_model_group_info_lists_tiered_pricing_ranges(local_model_cost_map: None) -> None: tiers: Final = ( {"range": [0, 256000], "input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}, diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx index f9f15f3efe3..08ee05aeb04 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx @@ -38,6 +38,7 @@ describe("DeploymentPricingBreakdown", () => { expect(pricesOf(azure, "Base prices", "Input / 1M tokens")).toEqual(["$2.75"]); expect(within(azure).queryByText(/Blank cells/)).not.toBeInTheDocument(); expect(screen.getByText(/This model has 2 deployments/)).toBeInTheDocument(); + expect(screen.getAllByText(/Adjustments made per request/)).toHaveLength(1); }); it("lays prices out by service tier column and context threshold group", () => { @@ -45,11 +46,11 @@ describe("DeploymentPricingBreakdown", () => { const openai = screen.getByRole("region", { name: "OpenAI pricing" }); const header = within(openai).getAllByRole("rowgroup")[0]; - expect(within(header).getAllByRole("columnheader").map((cell) => cell.textContent)).toEqual([ - "Charge", - "Standard", - "Priority", - ]); + expect( + within(header) + .getAllByRole("columnheader") + .map((cell) => cell.textContent), + ).toEqual(["Charge", "Standard", "Priority"]); expect(pricesOf(openai, "Base prices", "Output / 1M tokens")).toEqual(["$10.00", ""]); expect(pricesOf(openai, "Above 272K tokens", "Cache read / 1M tokens")).toEqual(["", "$1.00"]); expect(within(openai).getByText("Blank cells aren't priced separately for this deployment.")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx index 96f047e9d1b..3d967e1d43e 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx @@ -133,6 +133,7 @@ export function DeploymentPricingBreakdown({ {deployments.map((deployment, index) => ( ))} +

{t("deploymentPricing.requestAdjustments")}

); } diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts index 5c1f2c347e2..4e38f55d1b3 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts @@ -8,6 +8,7 @@ describe("parseCostKey", () => { ["output_cost_per_token_flex", { charge: "output", unit: "token", threshold: null, tier: "flex" }], ["input_cost_per_token_batches", { charge: "input", tier: "batches" }], ["cache_read_input_token_cost", { charge: "cache_read", unit: "token", tier: "standard" }], + ["explicit_cache_read_input_token_cost", { charge: "explicit_cache_read", unit: "token", tier: "standard" }], [ "cache_read_input_token_cost_above_272k_tokens_priority", { charge: "cache_read", threshold: 272_000, tier: "priority" }, @@ -86,6 +87,18 @@ describe("buildPricingTable", () => { ]); }); + it("shows an explicit cached read per 1M tokens on its own row", () => { + const { groups, unknown } = buildPricingTable({ + cache_read_input_token_cost: 1e-7, + explicit_cache_read_input_token_cost: 5e-8, + }); + expect(unknown).toEqual([]); + expect(groups[0].rows).toEqual([ + { label: "Cache read / 1M tokens", prices: { standard: "$0.10" } }, + { label: "Explicit cache read / 1M tokens", prices: { standard: "$0.05" } }, + ]); + }); + it("answers a cached read on priority above a threshold from its own group", () => { expect(table.groups[2].rows).toEqual([{ label: "Cache read / 1M tokens", prices: { priority: "$1.00" } }]); }); diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts index 940dc198f2b..337a8900b7d 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts @@ -21,7 +21,7 @@ export const SERVICE_TIER_LABELS: Readonly> = { const UNITS = ["token", "second", "image", "pixel", "character", "query", "request"] as const; type Unit = (typeof UNITS)[number]; -const CHARGES = ["input", "cache_read", "cache_write", "output"] as const; +const CHARGES = ["input", "cache_read", "explicit_cache_read", "cache_write", "output"] as const; type Charge = (typeof CHARGES)[number]; const MODALITIES = ["", "text", "image", "audio", "video", "reasoning"] as const; @@ -43,7 +43,7 @@ const TIER_SUFFIX = /^(.+)_(flex|balanced|priority|ultrafast|batches)$/; const THRESHOLD_SUFFIX = /^(.+)_above_(\d+)k_tokens$/; const CACHE_DURATION_SUFFIX = /^(.+)_above_(\d+hr)$/; const DIRECT_PRICE = /^(input|output)_cost_per_(?:(text|image|audio|video|reasoning)_(?:per_)?)?([a-z]+)$/; -const CACHE_PRICE = /^cache_(read|creation)_input_(?:(image|audio)_)?token_cost$/; +const CACHE_PRICE = /^(explicit_)?cache_(read|creation)_input_(?:(image|audio)_)?token_cost$/; const splitSuffix = (key: string, pattern: RegExp): readonly [string, string | null] => { const match = pattern.exec(key); @@ -60,8 +60,10 @@ const parseBase = (base: string): Pick, "cha } const cache = CACHE_PRICE.exec(base); if (cache) { - const [, kind, modality = ""] = cache; - return { charge: kind === "read" ? "cache_read" : "cache_write", modality: modality as Modality, unit: "token" }; + const [, explicit, kind, modality = ""] = cache; + if (kind === "creation") + return explicit ? null : { charge: "cache_write", modality: modality as Modality, unit: "token" }; + return { charge: explicit ? "explicit_cache_read" : "cache_read", modality: modality as Modality, unit: "token" }; } return null; }; @@ -89,12 +91,14 @@ const CHARGE_LABELS: Readonly Date: Thu, 5 Nov 2026 19:35:22 -0800 Subject: [PATCH 4/6] fix(model-hub): show prices written as text the way billing reads them A price written in config.yaml as 3e-7 arrives as text. Billing turns it into a number, but the hub dropped it, so a deployment showed no price or the cost map's price instead of its own. The hub now reads numeric text for regular prices and size-based price ranges, and skips text that isn't a number --- litellm/router_utils/model_group_pricing.py | 11 +++++++ tests/unit/test_router/test_router.py | 32 +++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/litellm/router_utils/model_group_pricing.py b/litellm/router_utils/model_group_pricing.py index b4ee63c3ef4..afadbc3eb38 100644 --- a/litellm/router_utils/model_group_pricing.py +++ b/litellm/router_utils/model_group_pricing.py @@ -1,3 +1,4 @@ +import math from collections.abc import Mapping from typing import Final @@ -10,12 +11,22 @@ from litellm.types.router import DeploymentPricing, PriceTier _RAW_PRICE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +def _numeric_text(value: str) -> float | None: + try: + parsed: Final = float(value) + except ValueError: + return None + return parsed if math.isfinite(parsed) else None + + def _cost_value(value: object) -> float | None: match value: case bool(): return None case int() | float(): return float(value) + case str(): + return _numeric_text(value) case _: return None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 5470f2bbf06..1b76546963b 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2108,6 +2108,38 @@ def test_model_group_info_lists_tiered_pricing_ranges(local_model_cost_map: None ] +def test_model_group_info_reads_prices_written_as_text_the_way_billing_does(local_model_cost_map: None) -> None: + tier: Final = {"range": [0, 256000], "input_cost_per_token": "5e-08", "output_cost_per_token": "4e-07"} + model: Final = _cost_map_model( + "hub-text-priced-model", {"tiered_pricing": [tier], "cache_read_input_token_cost": "N/A"} + ) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "flat", + "litellm_params": {"model": model, "input_cost_per_token": "3e-7", "output_cost_per_token": "1.2e-6"}, + "model_info": {"id": "hub-text-override"}, + }, + ] + ) + + tiered: Final = router.registry().group("tiered") + flat: Final = router.registry().group("flat") + + assert tiered is not None + assert flat is not None + assert [dict(t.costs) for t in tiered.deployment_pricing[0].tiered_pricing] == [ + {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07} + ] + assert "cache_read_input_token_cost" not in tiered.deployment_pricing[0].costs + flat_costs: Final = flat.deployment_pricing[0].costs + shown: Final = ("input_cost_per_token", "output_cost_per_token") + assert {key: flat_costs[key] for key in shown} == pytest.approx( + {key: _billed_rate(model, "hub-text-override", *_usage_billed_at(key)) for key in shown} + ) + + def test_model_group_info_lists_custom_prices_of_an_unmapped_deployment_and_skips_an_unpriced_one() -> None: router: Final = litellm.Router( model_list=[ From 5b38129816c1880a6a634af81a30d5903f05bd4d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 5 Nov 2026 19:47:49 -0800 Subject: [PATCH 5/6] style(ui): format public model hub test --- ui/litellm-dashboard/src/components/public_model_hub.test.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index ed58cc244e7..4081751ca73 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -347,7 +347,9 @@ describe("PublicModelHub", () => { const openai = await screen.findByRole("region", { name: "OpenAI pricing" }); const aboveThreshold = within(openai).getByRole("rowgroup", { name: "Above 272K tokens" }); - expect(within(aboveThreshold).getByRole("row", { name: /Cache read/ })).toHaveTextContent("Cache read / 1M tokens$1.00"); + expect(within(aboveThreshold).getByRole("row", { name: /Cache read/ })).toHaveTextContent( + "Cache read / 1M tokens$1.00", + ); expect(within(openai).getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); expect(within(screen.getByRole("dialog")).getByText("$2.5000")).toBeInTheDocument(); }); From 71fa6602438e7ed0ba034e58615734e40faf591a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 5 Nov 2026 19:57:43 -0800 Subject: [PATCH 6/6] test(model-hub): name the deployment pricing fixture to stay within the inline object lint budget --- .../src/components/AIHub/ModelHubTable.test.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 4132f7986c7..1b30b060ab0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -427,7 +427,7 @@ describe("ModelHubTable", () => { }); it("shows each deployment's own prices, by service tier, in the model details", async () => { - const dialog = await openDetails({ + const gpt5OnOpenAIAndAzure = { model_group: "gpt-5", providers: ["openai", "azure"], mode: "chat", @@ -439,7 +439,8 @@ describe("ModelHubTable", () => { }, { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }, ], - }); + }; + const dialog = await openDetails(gpt5OnOpenAIAndAzure); const openai = within(dialog).getByRole("region", { name: "OpenAI pricing" }); expect(within(openai).getByRole("row", { name: /Input \/ 1M tokens/ })).toHaveTextContent("$2.50$5.00");