diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e857461785c..3d14702fbd6 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Collection, Iterable, Mapping from types import MappingProxyType @@ -195,6 +196,31 @@ def coerce_token_limit(value: object) -> int | None: return None +def coerce_token_price(value: object) -> float | None: + """ + Coerce a per-token price to a float, treating a malformed value as absent. + + Prices reach the /v1/models listing from the same uncoerced sources as token limits + (a deployment's model_info is registered into litellm.model_cost verbatim), so a + config value like "0.000003" has to survive while "" does not fail the listing. + + Negative and non-finite values are rejected: a caller pricing a request would read a + negative rate as a discount, and NaN/inf serialize to invalid JSON. Bools are rejected + because True/False is never a meaningful price. + """ + if isinstance(value, bool): + return None + if not isinstance(value, (int, float, str)): + return None + try: + price: Final = float(value) + except (TypeError, ValueError, OverflowError): + return None + if not math.isfinite(price) or price < 0: + return None + return price + + _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = { # Anthropic "stop_sequence": "stop", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 000ffa9c305..640a10074a3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11853,6 +11853,7 @@ async def model_list( scope: str | None = None, healthy_only: bool | None = False, client_version: str | None = None, + include_pricing: bool | None = False, ): """ Use `/model/info` - to get detailed model information, example - pricing, mode, etc. @@ -11866,6 +11867,13 @@ async def model_list( each model's `model_info.service_tiers` as its service tiers; absent, the OpenAI shape below is returned - include_metadata: Include additional metadata in the response with fallback information + - include_pricing: Include a `pricing` object per model, carrying + `input_cost_per_token` and `output_cost_per_token` in USD per token. + Custom pricing configured on a deployment takes precedence over the + built-in cost map. A price the proxy does not know is reported as + `null`, never 0, so an unmapped model is not read as free. When a + model group's deployments disagree on price, the highest is + reported, matching `/model_group/info`. - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") Defaults to "general" when include_metadata=true - scope: Optional scope parameter. Currently only accepts "expand". @@ -12005,6 +12013,7 @@ async def model_list( include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, + include_pricing=include_pricing or False, ) model_info["id"] = response_id model_data.append(model_info) @@ -12073,6 +12082,7 @@ async def model_list( include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, + include_pricing=include_pricing or False, ) model_info["id"] = response_id model_data.append(model_info) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3e73e2dee40..859a0dd0348 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -81,7 +81,8 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.proxy.model_listing import ModelInfoResponse +from litellm.types.proxy.model_listing import ModelInfoResponse, ModelPricing +from litellm.types.router import DeploymentListingPrice from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage try: @@ -143,6 +144,7 @@ from litellm.integrations.SlackAlerting.utils import add_langfuse_trace_id_to_al from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, + coerce_token_price, get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, @@ -286,6 +288,7 @@ if TYPE_CHECKING: from litellm.repositories.prisma_protocols import TableActions from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline + from litellm.types.router import DeploymentModelListingInfo Span = _Span | object else: @@ -8952,6 +8955,138 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: return max(limits) if limits else None +def _declared_price(info: ModelInfo, field: str, cost_map: Mapping[str, object]) -> float | None: + """The price ``field`` carries, but only when the cost map actually declares it. + + A resolved ``ModelInfo`` cannot be trusted on its own here. Once the router registers + a sparse cost-map entry for a deployment the catalog has never priced, the lookup + stops raising and defaults the missing costs to 0, so an unpriced model would be + served as free. The raw entry is the authority on whether a price exists at all, + which is the same test ``_is_cost_explicitly_configured`` makes for budget checks. + + A model the catalog deliberately prices at 0.0 still reports 0.0; one it never priced + reports None. + """ + key: Final = info.get("key") + raw: Final = cost_map.get(key) if isinstance(key, str) else None + if not isinstance(raw, Mapping) or field not in raw: + return None + return coerce_token_price(info.get(field, raw[field])) + + +def _first_price(candidates: tuple[ModelInfo, ...], field: str, cost_map: Mapping[str, object]) -> float | None: + """The price ``field`` takes from the best cost-map entry describing one deployment. + + ``candidates`` is best-source-first (the exact entry for the deployment's underlying + model before a generalized family fallback), so these are not competing prices to + compare: it is one price described with varying reliability, and the first usable + answer wins. + """ + return next( + (price for price in (_declared_price(info, field, cost_map) for info in candidates) if price is not None), + None, + ) + + +def _listing_deployment_prices( + lookup_model: str, + listing_info: "DeploymentModelListingInfo | None", + llm_router: Optional["Router"], +) -> tuple["DeploymentListingPrice", ...]: + """The per-deployment prices behind a listed name, one entry per deployment. + + A name the router indexes carries its deployments' own records. A wildcard-expanded + name has none, so the overrides live on the patterns it expands from and are asked for + there, one record per matched deployment because a request can route to any of them. + Those records are keyed to None so a deployment that configures nothing, or configures + only one half, still falls back to the catalog entry the listing already resolved. + """ + if listing_info is not None: + return listing_info.deployment_prices + + wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else () + if not wildcard_prices: + return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),) + return tuple( + DeploymentListingPrice( + cost_map_key=None, + input_cost_per_token=price.input_cost_per_token, + output_cost_per_token=price.output_cost_per_token, + ) + for price in wildcard_prices + ) + + +def _deployment_effective_price( + configured: float | None, + cost_map_key: str | None, + field: str, + candidates_by_key: Mapping[str | None, tuple[ModelInfo, ...]], + cost_map: Mapping[str, object], +) -> float | None: + """What one deployment charges for ``field``: its own override, else its catalog entry.""" + if configured is not None: + return configured + return _first_price(candidates_by_key.get(cost_map_key, ()), field, cost_map) + + +def _highest_price(prices: tuple[float | None, ...]) -> float | None: + known: Final = tuple(price for price in prices if price is not None) + return max(known) if known else None + + +def _listing_pricing( + candidate_sets: tuple[tuple[ModelInfo, ...], ...], + deployment_models: tuple[str | None, ...], + deployment_prices: tuple["DeploymentListingPrice", ...], + cost_map: Mapping[str, object], +) -> ModelPricing: + """Effective per-token prices for a listed model, in USD. + + Each deployment resolves its own price first, its override when it has one and its + catalog entry otherwise, and the group reports the highest of those. Comparing + overrides against each other and only then falling back to the catalog would hide a + dearer deployment that happens to configure no override, quoting a caller less than + the request that lands there actually costs. + + The highest is the deliberate pick across a group, matching what ``/model_group/info`` + reports, because the listing cannot know which deployment a later request will reach. + + A price the proxy does not know is reported as None rather than 0, so a caller doing + cost accounting cannot read an unmapped model as free. + """ + candidates_by_key: Final[Mapping[str | None, tuple[ModelInfo, ...]]] = MappingProxyType( + {key: candidates for key, candidates in zip(deployment_models, candidate_sets)} + ) + pricing: Final[ModelPricing] = { + "input_cost_per_token": _highest_price( + tuple( + _deployment_effective_price( + deployment.input_cost_per_token, + deployment.cost_map_key, + "input_cost_per_token", + candidates_by_key, + cost_map, + ) + for deployment in deployment_prices + ) + ), + "output_cost_per_token": _highest_price( + tuple( + _deployment_effective_price( + deployment.output_cost_per_token, + deployment.cost_map_key, + "output_cost_per_token", + candidates_by_key, + cost_map, + ) + for deployment in deployment_prices + ) + ), + } + return pricing + + def create_model_info_response( model_id: str, provider: str, @@ -8959,6 +9094,8 @@ def create_model_info_response( fallback_type: str | None = None, llm_router: Optional["Router"] = None, get_model_info: Callable[[str], ModelInfo] = litellm.get_model_info, + include_pricing: bool = False, + cost_map: Mapping[str, object] | None = None, ) -> ModelInfoResponse: """ Create a standardized OpenAI-compatible model object. @@ -8966,6 +9103,11 @@ def create_model_info_response( When include_metadata is true, attaches the model's configured fallbacks (resolved via the router under fallback_type, defaulting to "general"). Raises HTTPException(400) for an unknown fallback_type. + + When include_pricing is true, attaches the model's effective per-token prices in USD, + resolved from the same deployment-then-cost-map sources as the token limits above. + cost_map defaults to litellm.model_cost, resolved per call because a price reload + rebuilds it. """ from litellm.proxy.auth.model_checks import get_all_fallbacks @@ -9032,6 +9174,14 @@ def create_model_info_response( if max_output_tokens is not None: base["max_output_tokens"] = max_output_tokens + if include_pricing: + base["pricing"] = _listing_pricing( + candidate_sets, + deployment_models, + _listing_deployment_prices(lookup_model, listing_info, llm_router), + cost_map if cost_map is not None else litellm.model_cost, + ) + if not include_metadata: return base diff --git a/litellm/router.py b/litellm/router.py index 7bedb714338..4c8f3ff143b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -79,6 +79,7 @@ from litellm.integrations.otel.runtime import phase_attributes, phase_event, pha from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, + coerce_token_price, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, @@ -302,6 +303,7 @@ from litellm.types.router import ( CredentialLiteLLMParams, CustomRoutingStrategyBase, Deployment, + DeploymentListingPrice, DeploymentModelListingInfo, DeploymentTypedDict, DiscoveredDeploymentModelInfo, @@ -10796,6 +10798,10 @@ class Router: cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + deployment_prices=tuple( + self._deployment_listing_price(model_info, litellm_params) + for model_info, litellm_params in zip(model_infos, params) + ), ) @staticmethod @@ -10808,6 +10814,76 @@ class Router: ) return max(limits) if limits else None + @staticmethod + def _configured_price( + model_info: Mapping[str, Any], + litellm_params: Mapping[str, Any], + field: str, + ) -> float | None: + """One deployment's custom price for ``field``, or None when it configures none. + + Both sources are read because custom pricing is accepted in ``model_info`` and in + ``litellm_params``; the router itself copies the latter onto the former when it + builds a deployment's cost-map entry. ``litellm_params`` wins, matching that copy. + """ + from_params: Final = coerce_token_price(litellm_params.get(field)) + return from_params if from_params is not None else coerce_token_price(model_info.get(field)) + + @staticmethod + def _deployment_listing_price( + model_info: Mapping[str, Any], + litellm_params: Mapping[str, Any], + ) -> DeploymentListingPrice: + """What one deployment contributes to its listed name's price.""" + cost_map_key: Final = ( + model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") + ) + return DeploymentListingPrice( + cost_map_key=cost_map_key if isinstance(cost_map_key, str) and cost_map_key else None, + input_cost_per_token=Router._configured_price(model_info, litellm_params, "input_cost_per_token"), + output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"), + ) + + def _pattern_deployments(self) -> Iterator[Mapping[str, object]]: + """Every deployment registered behind a wildcard pattern, across all patterns.""" + for deployments in self.pattern_router.patterns.values(): + yield from deployments + + def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]: + """One record per wildcard deployment that a name expands from, in match order. + + Wildcard-expanded rows are absent from the model-name index, so their deployments' + overrides are invisible to ``get_model_listing_info`` and the listing would quote + the catalog while the request is billed at an override. + + Every matched deployment is returned rather than the first, because a request can + route to any of them: the caller resolves each one and reports the dearest, the same + way it treats a group the index does know. + + Pattern matching is the expensive path this listing exists to avoid, so it only + runs when a wildcard deployment actually configures a price: with none, the catalog + is already the right answer and nothing here can change it. + """ + if not any( + price.input_cost_per_token is not None or price.output_cost_per_token is not None + for price in ( + Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + for deployment in self._pattern_deployments() + ) + ): + return () + + return tuple( + Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + for deployment in (self.pattern_router.route(model_name) or ()) + ) + def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index 24cfa85eee4..6d2c6d0aaa6 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -2,13 +2,24 @@ from typing import Literal -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict class ModelInfoMetadata(TypedDict): fallbacks: list[str] +class ModelPricing(TypedDict): + """Effective token prices for a listed model, in USD per token. + + ``None`` is not the same as free: an unmapped model reports null so a caller doing + cost accounting cannot read a missing price as a zero-cost model. + """ + + input_cost_per_token: ReadOnly[float | None] + output_cost_per_token: ReadOnly[float | None] + + class ModelInfoResponse(TypedDict): """OpenAI-compatible model object. `mode`, `max_input_tokens`, and `max_output_tokens` are attached when the cost map or deployment config @@ -23,3 +34,4 @@ class ModelInfoResponse(TypedDict): max_input_tokens: NotRequired[int] max_output_tokens: NotRequired[int] metadata: NotRequired[ModelInfoMetadata] + pricing: NotRequired[ModelPricing] diff --git a/litellm/types/router.py b/litellm/types/router.py index a66c4571b39..27b0e9568ec 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -789,6 +789,22 @@ class DiscoveredDeploymentModelInfo: limits: Mapping[str, int] +@dataclass(frozen=True, slots=True) +class DeploymentListingPrice: + """What one deployment behind a listed name contributes to its price. + + ``cost_map_key`` names the catalog entry that deployment resolves to, so a caller can + fall back to the catalog for a deployment that configures no price of its own. The + prices are whatever that single deployment configures, never a group aggregate: a + group's price is only correct once each member resolves its own effective price + first, since a deployment without an override can still be the dearest one. + """ + + cost_map_key: str | None + input_cost_per_token: float | None + output_cost_per_token: float | None + + @dataclass(frozen=True, slots=True) class DeploymentModelListingInfo: """What the deployments behind a model name contribute to its OpenAI-compatible listing entry. @@ -800,11 +816,18 @@ class DeploymentModelListingInfo: order, so the ordinary group -- several interchangeable deployments of one model -- carries exactly one. The token limits are the widest explicitly set in any deployment's ``model_info``, which outrank anything the cost map says. + + ``deployment_prices`` carries one entry per deployment, in config order, rather than a + group aggregate: the cost map cannot supply custom pricing on its own, because a + deployment's override is registered only under its own model id and is deliberately + stripped from the shared backend key so one deployment's override never becomes + another's price. """ cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None + deployment_prices: tuple[DeploymentListingPrice, ...] = () class RouterErrors(enum.Enum): diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 1f716b41acd..ce87b8a7e87 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.core_helpers import ( RESPONSE_COST_HEADER, bind_budget_reservation_to_callbacks, budget_reservation_from_metadata, + coerce_token_price, drop_params_env_flag, drop_params_flag, get_parent_otel_span_from_kwargs, @@ -586,3 +587,53 @@ class TestProviderResponseHeadersInHiddenParams: assert get_provider_response_headers_from_hidden_params(sibling) is None assert "additional_headers" not in sibling._hidden_params + + +class TestCoerceTokenPrice: + """Per-token prices reach the listing from the same uncoerced sources as token + limits, since a deployment's model_info is registered into litellm.model_cost + verbatim. A config value written as a string has to survive, while anything that + cannot be a price has to read as absent rather than as free.""" + + @pytest.mark.parametrize( + "value, expected", + [ + (0.000003, 0.000003), + (3, 3.0), + ("0.000003", 0.000003), + ("3", 3.0), + (0, 0.0), + ("0", 0.0), + ], + ) + def test_a_usable_price_survives_as_a_float(self, value, expected): + assert coerce_token_price(value) == expected, f"{value!r} should coerce to {expected}" + + def test_zero_is_a_price_not_an_absence(self): + """A deployment priced at zero is deliberately free, which is not the same as + a model nobody has priced.""" + assert coerce_token_price(0) == 0.0, "an explicit zero must not read as absent" + assert coerce_token_price(0) is not None, "an explicit zero must not read as absent" + + @pytest.mark.parametrize("value", [True, False]) + def test_a_bool_is_rejected(self, value): + """True is an int in Python, so without this guard it would price a token at 1.""" + assert coerce_token_price(value) is None, f"{value!r} is never a meaningful price" + + @pytest.mark.parametrize("value", [None, [], {}, (), object(), b"3"]) + def test_a_value_that_is_not_a_number_or_string_is_rejected(self, value): + assert coerce_token_price(value) is None, f"{value!r} cannot be a price" + + @pytest.mark.parametrize("value", ["", "abc", "1.2.3", " "]) + def test_a_string_that_is_not_a_number_is_rejected(self, value): + assert coerce_token_price(value) is None, f"{value!r} must not fail the listing" + + @pytest.mark.parametrize("value", [-1, -0.5, "-0.000003"]) + def test_a_negative_price_is_rejected(self, value): + """A caller pricing a request would read a negative rate as a discount.""" + assert coerce_token_price(value) is None, f"{value!r} must not be reported as a price" + + @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), "nan", "inf"]) + def test_a_non_finite_price_is_rejected(self, value): + """NaN and infinity serialize to invalid JSON.""" + assert coerce_token_price(value) is None, f"{value!r} must not reach the response body" diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index ec7104fa459..bd33adf82dc 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -989,8 +989,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: from typing import cast import litellm +from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME from litellm.proxy.utils import create_model_info_response -from litellm.types.router import DeploymentModelListingInfo +from litellm.types.router import DeploymentListingPrice, DeploymentModelListingInfo from litellm.types.utils import ModelInfo @@ -1131,6 +1132,484 @@ def test_create_model_info_response_reports_widest_window_in_a_mixed_group(): assert response["max_output_tokens"] == 128000, keys +def test_create_model_info_response_omits_pricing_unless_asked(): + """A caller who does not ask for pricing gets the same entry as before the flag. + + Asserted whole rather than as "pricing is absent", so any other key the listing starts + emitting by default fails here too. + """ + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info( + input_cost_per_token=2.5e-06, output_cost_per_token=1e-05 + ), + ) + + assert response == { + "id": "gpt-4o", + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": "openai", + }, f"the default listing entry changed shape: {response}" + + +def test_create_model_info_response_includes_cost_map_pricing(): + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05 + ), + cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + }, f"expected the cost-map prices, got {response['pricing']}" + + +def test_create_model_info_response_reports_unpriced_model_as_null_not_zero(): + """A model the catalog never priced must report null even though the lookup says 0. + + Once the router registers a sparse cost-map entry for an unmapped deployment, the + lookup stops raising and defaults the missing costs to 0. Trusting that value served + an unpriced model as free, which is exactly what #35312 asks the listing not to do. + """ + response = create_model_info_response( + model_id="unknown-model", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="openai/some-model-that-does-not-exist", + input_cost_per_token=0, + output_cost_per_token=0, + ), + cost_map={"openai/some-model-that-does-not-exist": {"id": "deployment-hash"}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": None, + "output_cost_per_token": None, + }, f"a sparse cost-map entry must read as unknown, got {response['pricing']}" + + +def test_create_model_info_response_reports_declared_zero_price_as_zero(): + """A model the catalog deliberately prices at 0 is free, and must not report null. + + This is the counterpart to the unpriced case: both resolve to 0, only the raw cost-map + entry distinguishes "free" from "nobody ever priced this". + """ + response = create_model_info_response( + model_id="free-model", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="free-model", input_cost_per_token=0.0, output_cost_per_token=0.0 + ), + cost_map={"free-model": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, f"a catalog-declared zero is free, not unknown, got {response['pricing']}" + + +def test_create_model_info_response_configured_pricing_overrides_cost_map(): + """Custom pricing is the price the request is billed at, so it outranks the cost map. + + The cost map cannot carry it: the router strips custom pricing from the shared backend + key so one deployment's override never becomes another's price. + """ + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("gpt-4o",), + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=( + DeploymentListingPrice( + cost_map_key="gpt-4o", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06 + ), + ), + ) + + response = create_model_info_response( + model_id="house-gpt", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05 + ), + cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + }, f"configured pricing must outrank the cost map, got {response['pricing']}" + + +def test_create_model_info_response_reports_highest_price_in_a_mixed_group(): + """A group mixing models quotes the dearest member, matching /model_group/info. + + Quoting the cheapest would under-budget every request that lands on the dearest + deployment, and the listing cannot know which one a later request will reach. + """ + prices = { + "cheap-model": _fake_model_info( + key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07 + ), + "dear-model": _fake_model_info( + key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05 + ), + } + cost_map = { + "cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07}, + "dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05}, + } + + for keys in (("cheap-model", "dear-model"), ("dear-model", "cheap-model")): + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=keys, + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=tuple( + DeploymentListingPrice(cost_map_key=key, input_cost_per_token=None, output_cost_per_token=None) + for key in keys + ), + ) + + response = create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda model: prices[model], + cost_map=cost_map, + ) + + assert response["pricing"] == { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + }, keys + + +def test_create_model_info_response_does_not_let_a_custom_rate_hide_a_dearer_catalog_one(): + """One deployment's override must not stand in for a group that has a dearer member. + + Comparing overrides against each other and only then falling back to the catalog + quoted the cheap override here while a request routed to the uncustomised deployment + billed 17x that, and disagreed with /model_group/info. + """ + prices = { + "cheap-model": _fake_model_info(key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07), + "dear-model": _fake_model_info(key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05), + } + cost_map = { + "cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07}, + "dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05}, + } + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("cheap-model", "dear-model"), + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=( + DeploymentListingPrice( + cost_map_key="cheap-model", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06 + ), + DeploymentListingPrice(cost_map_key="dear-model", input_cost_per_token=None, output_cost_per_token=None), + ), + ) + + response = create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda model: prices[model], + cost_map=cost_map, + ) + + assert response["pricing"] == { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + }, f"the uncustomised deployment is dearer and must set the price, got {response['pricing']}" + + +def test_create_model_info_response_prefers_a_cheaper_override_over_its_own_catalog_price(): + """The override is what the request is billed at, so it wins even when it is cheaper. + + The guard against the case above must not turn into "always take the catalog". + """ + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("gpt-4o",), + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=( + DeploymentListingPrice(cost_map_key="gpt-4o", input_cost_per_token=3e-08, output_cost_per_token=4e-08), + ), + ) + + response = create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 3e-08, + "output_cost_per_token": 4e-08, + }, f"the override is what the request bills, even when cheaper, got {response['pricing']}" + + +def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): + """A wildcard-expanded name has no index entry, so its override lives on the pattern. + + Reading only the index quoted the catalog here while the request billed at the + wildcard's own rate. + """ + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9e-05, + }, f"expected the wildcard override, got {response['pricing']}" + + +def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern(): + """A pattern can front several deployments, and routing can pick any of them. + + Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing + advertised less than the request could cost. + """ + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ), + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05 + ), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 4e-05, + "output_cost_per_token": 9e-05, + }, f"expected the dearest matched deployment per field, got {response['pricing']}" + + +def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog(): + """A matched deployment configuring nothing is billed at the catalog, so it can be the + dearest member even when its sibling carries a very cheap override.""" + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08 + ), + DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + }, f"the sibling without an override bills catalog and is dearest, got {response['pricing']}" + + +def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row(): + """An unpriced wildcard changes nothing, so the catalog answer still stands.""" + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = () + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + }, f"an unpriced wildcard leaves the catalog answer, got {response['pricing']}" + + +def test_create_model_info_response_reports_unknown_price_as_null(): + """An unmapped model must not read as free to a caller doing cost accounting.""" + response = create_model_info_response( + model_id="my-custom-deployment", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=_raise_unmapped, + ) + + assert response["pricing"] == { + "input_cost_per_token": None, + "output_cost_per_token": None, + }, f"an unmapped model must read as unknown, got {response['pricing']}" + + +def test_create_model_info_response_ignores_malformed_price(): + """A malformed or negative price is dropped rather than served or blowing up the listing. + + model_info is registered into the cost map verbatim, so a config typo reaches here + uncoerced; a negative rate would read as a discount and NaN is not valid JSON. + """ + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token="", output_cost_per_token=-1e-06 + ), + cost_map={"gpt-4o": {"input_cost_per_token": "", "output_cost_per_token": -1e-06}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": None, + "output_cost_per_token": None, + }, f"malformed and negative values must be dropped, got {response['pricing']}" + + +def test_create_model_info_response_accepts_string_price(): + """A price configured as a string still reaches the listing as a number.""" + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token="0.0000025", output_cost_per_token="0.00001" + ), + cost_map={"gpt-4o": {"input_cost_per_token": "0.0000025", "output_cost_per_token": "0.00001"}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + }, f"a string price must reach the listing as a number, got {response['pricing']}" + + +def test_create_model_info_response_pricing_does_not_call_router_group_info(): + """Pricing must not pull /v1/models onto the expensive group-info path (#33721).""" + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = () + + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info(key="gpt-4o", input_cost_per_token=2.5e-06), + cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06}}, + ) + + router.get_model_group_info.assert_not_called() + assert response["pricing"]["input_cost_per_token"] == 2.5e-06, f"expected the cost-map price, got {response['pricing']}" + + +def test_get_model_listing_info_carries_custom_deployment_pricing(): + """Custom pricing set in litellm_params reaches the listing through the router index.""" + router = litellm.Router( + model_list=[ + { + "model_name": "house-gpt", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + }, + }, + ] + ) + + listing_info = router.get_model_listing_info("house-gpt") + + assert listing_info is not None + assert listing_info.deployment_prices == ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06 + ), + ) + + response = create_model_info_response( + model_id="house-gpt", + provider="openai", + llm_router=router, + include_pricing=True, + ) + + assert response["pricing"] == { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + }, f"expected the deployment's configured pricing, got {response['pricing']}" + + def test_create_model_info_response_resolves_alias_once_per_listing(): """The alias is the same for every deployment in the group, so it is looked up once.""" seen: list[str] = [] diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index cd490e127cf..581c2448df8 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -73,6 +73,7 @@ from litellm.types.llms.openai import ChatCompletionRequest from litellm.types.router import ( CustomRoutingStrategyBase, Deployment, + DeploymentListingPrice, DeploymentTypedDict, LiteLLM_Params, ModelGroupInfo, @@ -10512,6 +10513,176 @@ def test_get_model_listing_info_returns_none_for_unknown_name(): assert router.get_model_listing_info("not-a-real-model") is None +def test_configured_price_reads_model_info_when_litellm_params_has_none(): + """Custom pricing is accepted in both blocks, so model_info alone still counts.""" + price = litellm.Router._configured_price( + {"input_cost_per_token": 3e-07}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token" + ) + + assert price == 3e-07, f"expected the model_info price, got {price}" + + +def test_configured_price_is_none_when_the_deployment_configures_none(): + """Nothing configured means the cost map answers instead, so this must not report 0.""" + price = litellm.Router._configured_price( + {"mode": "chat"}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token" + ) + + assert price is None, f"expected no configured price, got {price}" + + +def test_configured_price_ignores_a_malformed_configured_value(): + """model_info reaches the listing uncoerced, so a config typo must not become a price.""" + price = litellm.Router._configured_price( + {"input_cost_per_token": ""}, {"input_cost_per_token": -1e-06}, "input_cost_per_token" + ) + + assert price is None, f"expected malformed and negative values to be dropped, got {price}" + + +def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params(): + """The record a deployment contributes: its catalog key, plus its own configured prices. + + base_model names the catalog entry when set, because litellm_params.model can be an + opaque backend name, and litellm_params wins over model_info for a price, matching the + copy the router makes when it builds the deployment's cost-map entry. + """ + price = litellm.Router._deployment_listing_price( + {"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06}, + {"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06}, + ) + + assert price == DeploymentListingPrice( + cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None + ), f"base_model keys the catalog and litellm_params wins the price, got {price}" + + +def test_get_model_listing_info_keeps_each_deployment_price_separate(): + """A group's price is only correct once each deployment resolves its own price first. + + Collapsing the group to the highest override here would hide a deployment that + configures nothing and is dearer in the catalog, so the listing gets one record per + deployment, carrying that deployment's own cost-map key. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "input_cost_per_token": 3e-07, + }, + }, + {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "test-key"}}, + ] + ) + + info = router.get_model_listing_info("mixed") + + assert info is not None + assert info.deployment_prices == ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=None + ), + DeploymentListingPrice(cost_map_key="openai/gpt-5.5", input_cost_per_token=None, output_cost_per_token=None), + ), f"each deployment needs its own record and key, got {info.deployment_prices}" + + +def test_pattern_deployments_yields_every_deployment_behind_every_pattern(): + """The wildcard price scan reads through this to decide whether any pattern + configures a price at all, so it has to see all of them: missing one would skip + pattern matching and quote the catalog for a deployment that overrides it.""" + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "k", + "input_cost_per_token": 9e-06, + }, + }, + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "k", + "input_cost_per_token": 4e-05, + }, + }, + { + "model_name": "anthropic/*", + "litellm_params": { + "model": "anthropic/*", + "api_key": "k", + "input_cost_per_token": 2e-05, + }, + }, + ] + ) + + seen = [ + deployment.get("litellm_params", {}).get("input_cost_per_token") + for deployment in router._pattern_deployments() + ] + + assert sorted(price for price in seen if price) == [ + 9e-06, + 2e-05, + 4e-05, + ], f"every deployment behind every pattern must be visited, saw {seen}" + + +def test_get_wildcard_listing_prices_reports_every_matched_deployment(): + """A pattern can front several deployments and routing can pick any of them, so the + listing needs all their prices, not the first one's: the caller reports the dearest.""" + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "test-key", + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9e-05, + }, + }, + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "test-key", + "input_cost_per_token": 4e-05, + "output_cost_per_token": 8e-05, + }, + }, + ] + ) + + prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini") + + assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == { + (9e-06, 9e-05), + (4e-05, 8e-05), + }, f"every matched deployment must be returned, got {prices}" + + +def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced(): + """Pattern matching is the expensive path, so an unpriced wildcard must not trigger it. + + With no override the catalog is already the right answer, so there is nothing to find. + """ + router = litellm.Router( + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}}] + ) + + with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed: + assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == () + + routed.assert_not_called() + + def test_get_model_listing_info_carries_configured_limits(): router = litellm.Router( model_list=[ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 769798b5134..08f030b0abb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10832,6 +10832,13 @@ export interface paths { * each model's `model_info.service_tiers` as its service tiers; absent, * the OpenAI shape below is returned * - include_metadata: Include additional metadata in the response with fallback information + * - include_pricing: Include a `pricing` object per model, carrying + * `input_cost_per_token` and `output_cost_per_token` in USD per token. + * Custom pricing configured on a deployment takes precedence over the + * built-in cost map. A price the proxy does not know is reported as + * `null`, never 0, so an unmapped model is not read as free. When a + * model group's deployments disagree on price, the highest is + * reported, matching `/model_group/info`. * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true * - scope: Optional scope parameter. Currently only accepts "expand". @@ -21542,6 +21549,13 @@ export interface paths { * each model's `model_info.service_tiers` as its service tiers; absent, * the OpenAI shape below is returned * - include_metadata: Include additional metadata in the response with fallback information + * - include_pricing: Include a `pricing` object per model, carrying + * `input_cost_per_token` and `output_cost_per_token` in USD per token. + * Custom pricing configured on a deployment takes precedence over the + * built-in cost map. A price the proxy does not know is reported as + * `null`, never 0, so an unmapped model is not read as free. When a + * model group's deployments disagree on price, the highest is + * reported, matching `/model_group/info`. * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true * - scope: Optional scope parameter. Currently only accepts "expand". @@ -66618,6 +66632,7 @@ export interface operations { scope?: string | null; healthy_only?: boolean | null; client_version?: string | null; + include_pricing?: boolean | null; }; header?: never; path?: never; @@ -80881,6 +80896,7 @@ export interface operations { scope?: string | null; healthy_only?: boolean | null; client_version?: string | null; + include_pricing?: boolean | null; }; header?: never; path?: never;