This commit is contained in:
Ajit Sharma S 2026-10-08 12:22:51 -04:00 • committed by GitHub
commit 85015e7f1f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 1017 additions and 3 deletions

View file

@ -2,6 +2,7 @@
## Helper utilities
import copy
import logging
import math
import re
from collections.abc import Collection, Iterable, Mapping
from types import MappingProxyType
@ -195,6 +196,31 @@ def coerce_token_limit(value: object) -> int | None:
return None
def coerce_token_price(value: object) -> float | None:
"""
Coerce a per-token price to a float, treating a malformed value as absent.
Prices reach the /v1/models listing from the same uncoerced sources as token limits
(a deployment's model_info is registered into litellm.model_cost verbatim), so a
config value like "0.000003" has to survive while "" does not fail the listing.
Negative and non-finite values are rejected: a caller pricing a request would read a
negative rate as a discount, and NaN/inf serialize to invalid JSON. Bools are rejected
because True/False is never a meaningful price.
"""
if isinstance(value, bool):
return None
if not isinstance(value, (int, float, str)):
return None
try:
price: Final = float(value)
except (TypeError, ValueError, OverflowError):
return None
if not math.isfinite(price) or price < 0:
return None
return price
_FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
# Anthropic
"stop_sequence": "stop",

View file

@ -11853,6 +11853,7 @@ async def model_list(
scope: str | None = None,
healthy_only: bool | None = False,
client_version: str | None = None,
include_pricing: bool | None = False,
):
"""
Use `/model/info` - to get detailed model information, example - pricing, mode, etc.
@ -11866,6 +11867,13 @@ async def model_list(
each model's `model_info.service_tiers` as its service tiers; absent,
the OpenAI shape below is returned
- include_metadata: Include additional metadata in the response with fallback information
- include_pricing: Include a `pricing` object per model, carrying
`input_cost_per_token` and `output_cost_per_token` in USD per token.
Custom pricing configured on a deployment takes precedence over the
built-in cost map. A price the proxy does not know is reported as
`null`, never 0, so an unmapped model is not read as free. When a
model group's deployments disagree on price, the highest is
reported, matching `/model_group/info`.
- fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
Defaults to "general" when include_metadata=true
- scope: Optional scope parameter. Currently only accepts "expand".
@ -12005,6 +12013,7 @@ async def model_list(
include_metadata=include_metadata or False,
fallback_type=fallback_type,
llm_router=llm_router,
include_pricing=include_pricing or False,
)
model_info["id"] = response_id
model_data.append(model_info)
@ -12073,6 +12082,7 @@ async def model_list(
include_metadata=include_metadata or False,
fallback_type=fallback_type,
llm_router=llm_router,
include_pricing=include_pricing or False,
)
model_info["id"] = response_id
model_data.append(model_info)

View file

@ -81,7 +81,8 @@ from litellm.proxy.common_utils.openai_error_payload import (
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.model_listing import ModelInfoResponse
from litellm.types.proxy.model_listing import ModelInfoResponse, ModelPricing
from litellm.types.router import DeploymentListingPrice
from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage
try:
@ -143,6 +144,7 @@ from litellm.integrations.SlackAlerting.utils import add_langfuse_trace_id_to_al
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
coerce_token_price,
get_or_create_metadata_bucket,
independent_snapshot,
is_expected_client_error,
@ -286,6 +288,7 @@ if TYPE_CHECKING:
from litellm.repositories.prisma_protocols import TableActions
from litellm.types.mcp_server.mcp_server_manager import MCPServer
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
from litellm.types.router import DeploymentModelListingInfo
Span = _Span | object
else:
@ -8952,6 +8955,138 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field:
return max(limits) if limits else None
def _declared_price(info: ModelInfo, field: str, cost_map: Mapping[str, object]) -> float | None:
"""The price ``field`` carries, but only when the cost map actually declares it.
A resolved ``ModelInfo`` cannot be trusted on its own here. Once the router registers
a sparse cost-map entry for a deployment the catalog has never priced, the lookup
stops raising and defaults the missing costs to 0, so an unpriced model would be
served as free. The raw entry is the authority on whether a price exists at all,
which is the same test ``_is_cost_explicitly_configured`` makes for budget checks.
A model the catalog deliberately prices at 0.0 still reports 0.0; one it never priced
reports None.
"""
key: Final = info.get("key")
raw: Final = cost_map.get(key) if isinstance(key, str) else None
if not isinstance(raw, Mapping) or field not in raw:
return None
return coerce_token_price(info.get(field, raw[field]))
def _first_price(candidates: tuple[ModelInfo, ...], field: str, cost_map: Mapping[str, object]) -> float | None:
"""The price ``field`` takes from the best cost-map entry describing one deployment.
``candidates`` is best-source-first (the exact entry for the deployment's underlying
model before a generalized family fallback), so these are not competing prices to
compare: it is one price described with varying reliability, and the first usable
answer wins.
"""
return next(
(price for price in (_declared_price(info, field, cost_map) for info in candidates) if price is not None),
None,
)
def _listing_deployment_prices(
lookup_model: str,
listing_info: "DeploymentModelListingInfo | None",
llm_router: Optional["Router"],
) -> tuple["DeploymentListingPrice", ...]:
"""The per-deployment prices behind a listed name, one entry per deployment.
A name the router indexes carries its deployments' own records. A wildcard-expanded
name has none, so the overrides live on the patterns it expands from and are asked for
there, one record per matched deployment because a request can route to any of them.
Those records are keyed to None so a deployment that configures nothing, or configures
only one half, still falls back to the catalog entry the listing already resolved.
"""
if listing_info is not None:
return listing_info.deployment_prices
wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else ()
if not wildcard_prices:
return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),)
return tuple(
DeploymentListingPrice(
cost_map_key=None,
input_cost_per_token=price.input_cost_per_token,
output_cost_per_token=price.output_cost_per_token,
)
for price in wildcard_prices
)
def _deployment_effective_price(
configured: float | None,
cost_map_key: str | None,
field: str,
candidates_by_key: Mapping[str | None, tuple[ModelInfo, ...]],
cost_map: Mapping[str, object],
) -> float | None:
"""What one deployment charges for ``field``: its own override, else its catalog entry."""
if configured is not None:
return configured
return _first_price(candidates_by_key.get(cost_map_key, ()), field, cost_map)
def _highest_price(prices: tuple[float | None, ...]) -> float | None:
known: Final = tuple(price for price in prices if price is not None)
return max(known) if known else None
def _listing_pricing(
candidate_sets: tuple[tuple[ModelInfo, ...], ...],
deployment_models: tuple[str | None, ...],
deployment_prices: tuple["DeploymentListingPrice", ...],
cost_map: Mapping[str, object],
) -> ModelPricing:
"""Effective per-token prices for a listed model, in USD.
Each deployment resolves its own price first, its override when it has one and its
catalog entry otherwise, and the group reports the highest of those. Comparing
overrides against each other and only then falling back to the catalog would hide a
dearer deployment that happens to configure no override, quoting a caller less than
the request that lands there actually costs.
The highest is the deliberate pick across a group, matching what ``/model_group/info``
reports, because the listing cannot know which deployment a later request will reach.
A price the proxy does not know is reported as None rather than 0, so a caller doing
cost accounting cannot read an unmapped model as free.
"""
candidates_by_key: Final[Mapping[str | None, tuple[ModelInfo, ...]]] = MappingProxyType(
{key: candidates for key, candidates in zip(deployment_models, candidate_sets)}
)
pricing: Final[ModelPricing] = {
"input_cost_per_token": _highest_price(
tuple(
_deployment_effective_price(
deployment.input_cost_per_token,
deployment.cost_map_key,
"input_cost_per_token",
candidates_by_key,
cost_map,
)
for deployment in deployment_prices
)
),
"output_cost_per_token": _highest_price(
tuple(
_deployment_effective_price(
deployment.output_cost_per_token,
deployment.cost_map_key,
"output_cost_per_token",
candidates_by_key,
cost_map,
)
for deployment in deployment_prices
)
),
}
return pricing
def create_model_info_response(
model_id: str,
provider: str,
@ -8959,6 +9094,8 @@ def create_model_info_response(
fallback_type: str | None = None,
llm_router: Optional["Router"] = None,
get_model_info: Callable[[str], ModelInfo] = litellm.get_model_info,
include_pricing: bool = False,
cost_map: Mapping[str, object] | None = None,
) -> ModelInfoResponse:
"""
Create a standardized OpenAI-compatible model object.
@ -8966,6 +9103,11 @@ def create_model_info_response(
When include_metadata is true, attaches the model's configured fallbacks
(resolved via the router under fallback_type, defaulting to "general").
Raises HTTPException(400) for an unknown fallback_type.
When include_pricing is true, attaches the model's effective per-token prices in USD,
resolved from the same deployment-then-cost-map sources as the token limits above.
cost_map defaults to litellm.model_cost, resolved per call because a price reload
rebuilds it.
"""
from litellm.proxy.auth.model_checks import get_all_fallbacks
@ -9032,6 +9174,14 @@ def create_model_info_response(
if max_output_tokens is not None:
base["max_output_tokens"] = max_output_tokens
if include_pricing:
base["pricing"] = _listing_pricing(
candidate_sets,
deployment_models,
_listing_deployment_prices(lookup_model, listing_info, llm_router),
cost_map if cost_map is not None else litellm.model_cost,
)
if not include_metadata:
return base

View file

@ -79,6 +79,7 @@ from litellm.integrations.otel.runtime import phase_attributes, phase_event, pha
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
coerce_token_price,
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
get_or_create_metadata_bucket,
@ -302,6 +303,7 @@ from litellm.types.router import (
CredentialLiteLLMParams,
CustomRoutingStrategyBase,
Deployment,
DeploymentListingPrice,
DeploymentModelListingInfo,
DeploymentTypedDict,
DiscoveredDeploymentModelInfo,
@ -10796,6 +10798,10 @@ class Router:
cost_map_keys=cost_map_keys,
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),
max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
deployment_prices=tuple(
self._deployment_listing_price(model_info, litellm_params)
for model_info, litellm_params in zip(model_infos, params)
),
)
@staticmethod
@ -10808,6 +10814,76 @@ class Router:
)
return max(limits) if limits else None
@staticmethod
def _configured_price(
model_info: Mapping[str, Any],
litellm_params: Mapping[str, Any],
field: str,
) -> float | None:
"""One deployment's custom price for ``field``, or None when it configures none.
Both sources are read because custom pricing is accepted in ``model_info`` and in
``litellm_params``; the router itself copies the latter onto the former when it
builds a deployment's cost-map entry. ``litellm_params`` wins, matching that copy.
"""
from_params: Final = coerce_token_price(litellm_params.get(field))
return from_params if from_params is not None else coerce_token_price(model_info.get(field))
@staticmethod
def _deployment_listing_price(
model_info: Mapping[str, Any],
litellm_params: Mapping[str, Any],
) -> DeploymentListingPrice:
"""What one deployment contributes to its listed name's price."""
cost_map_key: Final = (
model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
)
return DeploymentListingPrice(
cost_map_key=cost_map_key if isinstance(cost_map_key, str) and cost_map_key else None,
input_cost_per_token=Router._configured_price(model_info, litellm_params, "input_cost_per_token"),
output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"),
)
def _pattern_deployments(self) -> Iterator[Mapping[str, object]]:
"""Every deployment registered behind a wildcard pattern, across all patterns."""
for deployments in self.pattern_router.patterns.values():
yield from deployments
def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]:
"""One record per wildcard deployment that a name expands from, in match order.
Wildcard-expanded rows are absent from the model-name index, so their deployments'
overrides are invisible to ``get_model_listing_info`` and the listing would quote
the catalog while the request is billed at an override.
Every matched deployment is returned rather than the first, because a request can
route to any of them: the caller resolves each one and reports the dearest, the same
way it treats a group the index does know.
Pattern matching is the expensive path this listing exists to avoid, so it only
runs when a wildcard deployment actually configures a price: with none, the catalog
is already the right answer and nothing here can change it.
"""
if not any(
price.input_cost_per_token is not None or price.output_cost_per_token is not None
for price in (
Router._deployment_listing_price(
deployment.get("model_info") or MappingProxyType({}),
deployment.get("litellm_params") or MappingProxyType({}),
)
for deployment in self._pattern_deployments()
)
):
return ()
return tuple(
Router._deployment_listing_price(
deployment.get("model_info") or MappingProxyType({}),
deployment.get("litellm_params") or MappingProxyType({}),
)
for deployment in (self.pattern_router.route(model_name) or ())
)
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
"""
Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete

View file

@ -2,13 +2,24 @@
from typing import Literal
from typing_extensions import NotRequired, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
class ModelInfoMetadata(TypedDict):
fallbacks: list[str]
class ModelPricing(TypedDict):
"""Effective token prices for a listed model, in USD per token.
``None`` is not the same as free: an unmapped model reports null so a caller doing
cost accounting cannot read a missing price as a zero-cost model.
"""
input_cost_per_token: ReadOnly[float | None]
output_cost_per_token: ReadOnly[float | None]
class ModelInfoResponse(TypedDict):
"""OpenAI-compatible model object. `mode`, `max_input_tokens`, and
`max_output_tokens` are attached when the cost map or deployment config
@ -23,3 +34,4 @@ class ModelInfoResponse(TypedDict):
max_input_tokens: NotRequired[int]
max_output_tokens: NotRequired[int]
metadata: NotRequired[ModelInfoMetadata]
pricing: NotRequired[ModelPricing]

View file

@ -789,6 +789,22 @@ class DiscoveredDeploymentModelInfo:
limits: Mapping[str, int]
@dataclass(frozen=True, slots=True)
class DeploymentListingPrice:
"""What one deployment behind a listed name contributes to its price.
``cost_map_key`` names the catalog entry that deployment resolves to, so a caller can
fall back to the catalog for a deployment that configures no price of its own. The
prices are whatever that single deployment configures, never a group aggregate: a
group's price is only correct once each member resolves its own effective price
first, since a deployment without an override can still be the dearest one.
"""
cost_map_key: str | None
input_cost_per_token: float | None
output_cost_per_token: float | None
@dataclass(frozen=True, slots=True)
class DeploymentModelListingInfo:
"""What the deployments behind a model name contribute to its OpenAI-compatible listing entry.
@ -800,11 +816,18 @@ class DeploymentModelListingInfo:
order, so the ordinary group -- several interchangeable deployments of one model --
carries exactly one. The token limits are the widest explicitly set in any
deployment's ``model_info``, which outrank anything the cost map says.
``deployment_prices`` carries one entry per deployment, in config order, rather than a
group aggregate: the cost map cannot supply custom pricing on its own, because a
deployment's override is registered only under its own model id and is deliberately
stripped from the shared backend key so one deployment's override never becomes
another's price.
"""
cost_map_keys: tuple[str, ...]
max_input_tokens: int | None
max_output_tokens: int | None
deployment_prices: tuple[DeploymentListingPrice, ...] = ()
class RouterErrors(enum.Enum):

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.core_helpers import (
RESPONSE_COST_HEADER,
bind_budget_reservation_to_callbacks,
budget_reservation_from_metadata,
coerce_token_price,
drop_params_env_flag,
drop_params_flag,
get_parent_otel_span_from_kwargs,
@ -586,3 +587,53 @@ class TestProviderResponseHeadersInHiddenParams:
assert get_provider_response_headers_from_hidden_params(sibling) is None
assert "additional_headers" not in sibling._hidden_params
class TestCoerceTokenPrice:
"""Per-token prices reach the listing from the same uncoerced sources as token
limits, since a deployment's model_info is registered into litellm.model_cost
verbatim. A config value written as a string has to survive, while anything that
cannot be a price has to read as absent rather than as free."""
@pytest.mark.parametrize(
"value, expected",
[
(0.000003, 0.000003),
(3, 3.0),
("0.000003", 0.000003),
("3", 3.0),
(0, 0.0),
("0", 0.0),
],
)
def test_a_usable_price_survives_as_a_float(self, value, expected):
assert coerce_token_price(value) == expected, f"{value!r} should coerce to {expected}"
def test_zero_is_a_price_not_an_absence(self):
"""A deployment priced at zero is deliberately free, which is not the same as
a model nobody has priced."""
assert coerce_token_price(0) == 0.0, "an explicit zero must not read as absent"
assert coerce_token_price(0) is not None, "an explicit zero must not read as absent"
@pytest.mark.parametrize("value", [True, False])
def test_a_bool_is_rejected(self, value):
"""True is an int in Python, so without this guard it would price a token at 1."""
assert coerce_token_price(value) is None, f"{value!r} is never a meaningful price"
@pytest.mark.parametrize("value", [None, [], {}, (), object(), b"3"])
def test_a_value_that_is_not_a_number_or_string_is_rejected(self, value):
assert coerce_token_price(value) is None, f"{value!r} cannot be a price"
@pytest.mark.parametrize("value", ["", "abc", "1.2.3", " "])
def test_a_string_that_is_not_a_number_is_rejected(self, value):
assert coerce_token_price(value) is None, f"{value!r} must not fail the listing"
@pytest.mark.parametrize("value", [-1, -0.5, "-0.000003"])
def test_a_negative_price_is_rejected(self, value):
"""A caller pricing a request would read a negative rate as a discount."""
assert coerce_token_price(value) is None, f"{value!r} must not be reported as a price"
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), "nan", "inf"])
def test_a_non_finite_price_is_rejected(self, value):
"""NaN and infinity serialize to invalid JSON."""
assert coerce_token_price(value) is None, f"{value!r} must not reach the response body"

View file

@ -989,8 +989,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
from typing import cast
import litellm
from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME
from litellm.proxy.utils import create_model_info_response
from litellm.types.router import DeploymentModelListingInfo
from litellm.types.router import DeploymentListingPrice, DeploymentModelListingInfo
from litellm.types.utils import ModelInfo
@ -1131,6 +1132,484 @@ def test_create_model_info_response_reports_widest_window_in_a_mixed_group():
assert response["max_output_tokens"] == 128000, keys
def test_create_model_info_response_omits_pricing_unless_asked():
"""A caller who does not ask for pricing gets the same entry as before the flag.
Asserted whole rather than as "pricing is absent", so any other key the listing starts
emitting by default fails here too.
"""
response = create_model_info_response(
model_id="gpt-4o",
provider="openai",
llm_router=None,
get_model_info=lambda _model: _fake_model_info(
input_cost_per_token=2.5e-06, output_cost_per_token=1e-05
),
)
assert response == {
"id": "gpt-4o",
"object": "model",
"created": DEFAULT_MODEL_CREATED_AT_TIME,
"owned_by": "openai",
}, f"the default listing entry changed shape: {response}"
def test_create_model_info_response_includes_cost_map_pricing():
response = create_model_info_response(
model_id="gpt-4o",
provider="openai",
llm_router=None,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05
),
cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}},
)
assert response["pricing"] == {
"input_cost_per_token": 2.5e-06,
"output_cost_per_token": 1e-05,
}, f"expected the cost-map prices, got {response['pricing']}"
def test_create_model_info_response_reports_unpriced_model_as_null_not_zero():
"""A model the catalog never priced must report null even though the lookup says 0.
Once the router registers a sparse cost-map entry for an unmapped deployment, the
lookup stops raising and defaults the missing costs to 0. Trusting that value served
an unpriced model as free, which is exactly what #35312 asks the listing not to do.
"""
response = create_model_info_response(
model_id="unknown-model",
provider="openai",
llm_router=None,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="openai/some-model-that-does-not-exist",
input_cost_per_token=0,
output_cost_per_token=0,
),
cost_map={"openai/some-model-that-does-not-exist": {"id": "deployment-hash"}},
)
assert response["pricing"] == {
"input_cost_per_token": None,
"output_cost_per_token": None,
}, f"a sparse cost-map entry must read as unknown, got {response['pricing']}"
def test_create_model_info_response_reports_declared_zero_price_as_zero():
"""A model the catalog deliberately prices at 0 is free, and must not report null.
This is the counterpart to the unpriced case: both resolve to 0, only the raw cost-map
entry distinguishes "free" from "nobody ever priced this".
"""
response = create_model_info_response(
model_id="free-model",
provider="openai",
llm_router=None,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="free-model", input_cost_per_token=0.0, output_cost_per_token=0.0
),
cost_map={"free-model": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}},
)
assert response["pricing"] == {
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
}, f"a catalog-declared zero is free, not unknown, got {response['pricing']}"
def test_create_model_info_response_configured_pricing_overrides_cost_map():
"""Custom pricing is the price the request is billed at, so it outranks the cost map.
The cost map cannot carry it: the router strips custom pricing from the shared backend
key so one deployment's override never becomes another's price.
"""
router = MagicMock()
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
cost_map_keys=("gpt-4o",),
max_input_tokens=None,
max_output_tokens=None,
deployment_prices=(
DeploymentListingPrice(
cost_map_key="gpt-4o", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
),
),
)
response = create_model_info_response(
model_id="house-gpt",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05
),
cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}},
)
assert response["pricing"] == {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
}, f"configured pricing must outrank the cost map, got {response['pricing']}"
def test_create_model_info_response_reports_highest_price_in_a_mixed_group():
"""A group mixing models quotes the dearest member, matching /model_group/info.
Quoting the cheapest would under-budget every request that lands on the dearest
deployment, and the listing cannot know which one a later request will reach.
"""
prices = {
"cheap-model": _fake_model_info(
key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07
),
"dear-model": _fake_model_info(
key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05
),
}
cost_map = {
"cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07},
"dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05},
}
for keys in (("cheap-model", "dear-model"), ("dear-model", "cheap-model")):
router = MagicMock()
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
cost_map_keys=keys,
max_input_tokens=None,
max_output_tokens=None,
deployment_prices=tuple(
DeploymentListingPrice(cost_map_key=key, input_cost_per_token=None, output_cost_per_token=None)
for key in keys
),
)
response = create_model_info_response(
model_id="house-model",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda model: prices[model],
cost_map=cost_map,
)
assert response["pricing"] == {
"input_cost_per_token": 5e-06,
"output_cost_per_token": 3e-05,
}, keys
def test_create_model_info_response_does_not_let_a_custom_rate_hide_a_dearer_catalog_one():
"""One deployment's override must not stand in for a group that has a dearer member.
Comparing overrides against each other and only then falling back to the catalog
quoted the cheap override here while a request routed to the uncustomised deployment
billed 17x that, and disagreed with /model_group/info.
"""
prices = {
"cheap-model": _fake_model_info(key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07),
"dear-model": _fake_model_info(key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05),
}
cost_map = {
"cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07},
"dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05},
}
router = MagicMock()
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
cost_map_keys=("cheap-model", "dear-model"),
max_input_tokens=None,
max_output_tokens=None,
deployment_prices=(
DeploymentListingPrice(
cost_map_key="cheap-model", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
),
DeploymentListingPrice(cost_map_key="dear-model", input_cost_per_token=None, output_cost_per_token=None),
),
)
response = create_model_info_response(
model_id="house-model",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda model: prices[model],
cost_map=cost_map,
)
assert response["pricing"] == {
"input_cost_per_token": 5e-06,
"output_cost_per_token": 3e-05,
}, f"the uncustomised deployment is dearer and must set the price, got {response['pricing']}"
def test_create_model_info_response_prefers_a_cheaper_override_over_its_own_catalog_price():
"""The override is what the request is billed at, so it wins even when it is cheaper.
The guard against the case above must not turn into "always take the catalog".
"""
router = MagicMock()
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
cost_map_keys=("gpt-4o",),
max_input_tokens=None,
max_output_tokens=None,
deployment_prices=(
DeploymentListingPrice(cost_map_key="gpt-4o", input_cost_per_token=3e-08, output_cost_per_token=4e-08),
),
)
response = create_model_info_response(
model_id="house-model",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 3e-08,
"output_cost_per_token": 4e-08,
}, f"the override is what the request bills, even when cheaper, got {response['pricing']}"
def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row():
"""A wildcard-expanded name has no index entry, so its override lives on the pattern.
Reading only the index quoted the catalog here while the request billed at the
wildcard's own rate.
"""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
),
)
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 9e-06,
"output_cost_per_token": 9e-05,
}, f"expected the wildcard override, got {response['pricing']}"
def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern():
"""A pattern can front several deployments, and routing can pick any of them.
Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing
advertised less than the request could cost.
"""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
),
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05
),
)
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 4e-05,
"output_cost_per_token": 9e-05,
}, f"expected the dearest matched deployment per field, got {response['pricing']}"
def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog():
"""A matched deployment configuring nothing is billed at the catalog, so it can be the
dearest member even when its sibling carries a very cheap override."""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08
),
DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None),
)
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
}, f"the sibling without an override bills catalog and is dearest, got {response['pricing']}"
def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row():
"""An unpriced wildcard changes nothing, so the catalog answer still stands."""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = ()
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
}, f"an unpriced wildcard leaves the catalog answer, got {response['pricing']}"
def test_create_model_info_response_reports_unknown_price_as_null():
"""An unmapped model must not read as free to a caller doing cost accounting."""
response = create_model_info_response(
model_id="my-custom-deployment",
provider="openai",
llm_router=None,
include_pricing=True,
get_model_info=_raise_unmapped,
)
assert response["pricing"] == {
"input_cost_per_token": None,
"output_cost_per_token": None,
}, f"an unmapped model must read as unknown, got {response['pricing']}"
def test_create_model_info_response_ignores_malformed_price():
"""A malformed or negative price is dropped rather than served or blowing up the listing.
model_info is registered into the cost map verbatim, so a config typo reaches here
uncoerced; a negative rate would read as a discount and NaN is not valid JSON.
"""
response = create_model_info_response(
model_id="gpt-4o",
provider="openai",
llm_router=None,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o", input_cost_per_token="", output_cost_per_token=-1e-06
),
cost_map={"gpt-4o": {"input_cost_per_token": "", "output_cost_per_token": -1e-06}},
)
assert response["pricing"] == {
"input_cost_per_token": None,
"output_cost_per_token": None,
}, f"malformed and negative values must be dropped, got {response['pricing']}"
def test_create_model_info_response_accepts_string_price():
"""A price configured as a string still reaches the listing as a number."""
response = create_model_info_response(
model_id="gpt-4o",
provider="openai",
llm_router=None,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o", input_cost_per_token="0.0000025", output_cost_per_token="0.00001"
),
cost_map={"gpt-4o": {"input_cost_per_token": "0.0000025", "output_cost_per_token": "0.00001"}},
)
assert response["pricing"] == {
"input_cost_per_token": 2.5e-06,
"output_cost_per_token": 1e-05,
}, f"a string price must reach the listing as a number, got {response['pricing']}"
def test_create_model_info_response_pricing_does_not_call_router_group_info():
"""Pricing must not pull /v1/models onto the expensive group-info path (#33721)."""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = ()
response = create_model_info_response(
model_id="gpt-4o",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(key="gpt-4o", input_cost_per_token=2.5e-06),
cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06}},
)
router.get_model_group_info.assert_not_called()
assert response["pricing"]["input_cost_per_token"] == 2.5e-06, f"expected the cost-map price, got {response['pricing']}"
def test_get_model_listing_info_carries_custom_deployment_pricing():
"""Custom pricing set in litellm_params reaches the listing through the router index."""
router = litellm.Router(
model_list=[
{
"model_name": "house-gpt",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "test-key",
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
},
},
]
)
listing_info = router.get_model_listing_info("house-gpt")
assert listing_info is not None
assert listing_info.deployment_prices == (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
),
)
response = create_model_info_response(
model_id="house-gpt",
provider="openai",
llm_router=router,
include_pricing=True,
)
assert response["pricing"] == {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
}, f"expected the deployment's configured pricing, got {response['pricing']}"
def test_create_model_info_response_resolves_alias_once_per_listing():
"""The alias is the same for every deployment in the group, so it is looked up once."""
seen: list[str] = []

View file

@ -73,6 +73,7 @@ from litellm.types.llms.openai import ChatCompletionRequest
from litellm.types.router import (
CustomRoutingStrategyBase,
Deployment,
DeploymentListingPrice,
DeploymentTypedDict,
LiteLLM_Params,
ModelGroupInfo,
@ -10512,6 +10513,176 @@ def test_get_model_listing_info_returns_none_for_unknown_name():
assert router.get_model_listing_info("not-a-real-model") is None
def test_configured_price_reads_model_info_when_litellm_params_has_none():
"""Custom pricing is accepted in both blocks, so model_info alone still counts."""
price = litellm.Router._configured_price(
{"input_cost_per_token": 3e-07}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token"
)
assert price == 3e-07, f"expected the model_info price, got {price}"
def test_configured_price_is_none_when_the_deployment_configures_none():
"""Nothing configured means the cost map answers instead, so this must not report 0."""
price = litellm.Router._configured_price(
{"mode": "chat"}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token"
)
assert price is None, f"expected no configured price, got {price}"
def test_configured_price_ignores_a_malformed_configured_value():
"""model_info reaches the listing uncoerced, so a config typo must not become a price."""
price = litellm.Router._configured_price(
{"input_cost_per_token": ""}, {"input_cost_per_token": -1e-06}, "input_cost_per_token"
)
assert price is None, f"expected malformed and negative values to be dropped, got {price}"
def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params():
"""The record a deployment contributes: its catalog key, plus its own configured prices.
base_model names the catalog entry when set, because litellm_params.model can be an
opaque backend name, and litellm_params wins over model_info for a price, matching the
copy the router makes when it builds the deployment's cost-map entry.
"""
price = litellm.Router._deployment_listing_price(
{"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06},
{"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06},
)
assert price == DeploymentListingPrice(
cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None
), f"base_model keys the catalog and litellm_params wins the price, got {price}"
def test_get_model_listing_info_keeps_each_deployment_price_separate():
"""A group's price is only correct once each deployment resolves its own price first.
Collapsing the group to the highest override here would hide a deployment that
configures nothing and is dearer in the catalog, so the listing gets one record per
deployment, carrying that deployment's own cost-map key.
"""
router = litellm.Router(
model_list=[
{
"model_name": "mixed",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "test-key",
"input_cost_per_token": 3e-07,
},
},
{"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "test-key"}},
]
)
info = router.get_model_listing_info("mixed")
assert info is not None
assert info.deployment_prices == (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=None
),
DeploymentListingPrice(cost_map_key="openai/gpt-5.5", input_cost_per_token=None, output_cost_per_token=None),
), f"each deployment needs its own record and key, got {info.deployment_prices}"
def test_pattern_deployments_yields_every_deployment_behind_every_pattern():
"""The wildcard price scan reads through this to decide whether any pattern
configures a price at all, so it has to see all of them: missing one would skip
pattern matching and quote the catalog for a deployment that overrides it."""
router = litellm.Router(
model_list=[
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "k",
"input_cost_per_token": 9e-06,
},
},
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "k",
"input_cost_per_token": 4e-05,
},
},
{
"model_name": "anthropic/*",
"litellm_params": {
"model": "anthropic/*",
"api_key": "k",
"input_cost_per_token": 2e-05,
},
},
]
)
seen = [
deployment.get("litellm_params", {}).get("input_cost_per_token")
for deployment in router._pattern_deployments()
]
assert sorted(price for price in seen if price) == [
9e-06,
2e-05,
4e-05,
], f"every deployment behind every pattern must be visited, saw {seen}"
def test_get_wildcard_listing_prices_reports_every_matched_deployment():
"""A pattern can front several deployments and routing can pick any of them, so the
listing needs all their prices, not the first one's: the caller reports the dearest."""
router = litellm.Router(
model_list=[
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "test-key",
"input_cost_per_token": 9e-06,
"output_cost_per_token": 9e-05,
},
},
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "test-key",
"input_cost_per_token": 4e-05,
"output_cost_per_token": 8e-05,
},
},
]
)
prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini")
assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == {
(9e-06, 9e-05),
(4e-05, 8e-05),
}, f"every matched deployment must be returned, got {prices}"
def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced():
"""Pattern matching is the expensive path, so an unpriced wildcard must not trigger it.
With no override the catalog is already the right answer, so there is nothing to find.
"""
router = litellm.Router(
model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}}]
)
with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed:
assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == ()
routed.assert_not_called()
def test_get_model_listing_info_carries_configured_limits():
router = litellm.Router(
model_list=[

View file

@ -10832,6 +10832,13 @@ export interface paths {
* each model's `model_info.service_tiers` as its service tiers; absent,
* the OpenAI shape below is returned
* - include_metadata: Include additional metadata in the response with fallback information
* - include_pricing: Include a `pricing` object per model, carrying
* `input_cost_per_token` and `output_cost_per_token` in USD per token.
* Custom pricing configured on a deployment takes precedence over the
* built-in cost map. A price the proxy does not know is reported as
* `null`, never 0, so an unmapped model is not read as free. When a
* model group's deployments disagree on price, the highest is
* reported, matching `/model_group/info`.
* - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
* Defaults to "general" when include_metadata=true
* - scope: Optional scope parameter. Currently only accepts "expand".
@ -21542,6 +21549,13 @@ export interface paths {
* each model's `model_info.service_tiers` as its service tiers; absent,
* the OpenAI shape below is returned
* - include_metadata: Include additional metadata in the response with fallback information
* - include_pricing: Include a `pricing` object per model, carrying
* `input_cost_per_token` and `output_cost_per_token` in USD per token.
* Custom pricing configured on a deployment takes precedence over the
* built-in cost map. A price the proxy does not know is reported as
* `null`, never 0, so an unmapped model is not read as free. When a
* model group's deployments disagree on price, the highest is
* reported, matching `/model_group/info`.
* - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
* Defaults to "general" when include_metadata=true
* - scope: Optional scope parameter. Currently only accepts "expand".
@ -66618,6 +66632,7 @@ export interface operations {
scope?: string | null;
healthy_only?: boolean | null;
client_version?: string | null;
include_pricing?: boolean | null;
};
header?: never;
path?: never;
@ -80881,6 +80896,7 @@ export interface operations {
scope?: string | null;
healthy_only?: boolean | null;
client_version?: string | null;
include_pricing?: boolean | null;
};
header?: never;
path?: never;