From 86c5d653faf83e268dc1072996275054804304b8 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Mon, 21 Sep 2026 09:38:07 +0530 Subject: [PATCH 1/9] feat(proxy): optional pricing in /v1/models listing Adds `?include_pricing=true` to /models and /v1/models, attaching a `pricing` object with `input_cost_per_token` and `output_cost_per_token` in USD per token. Default responses are unchanged. Prices resolve the same way the listing already resolves token limits: custom pricing configured on a deployment first, then the cost map. The cost map cannot supply custom pricing on its own, because the router strips it from the shared backend key so one deployment's override never becomes another's price. A group whose deployments disagree reports the highest price, matching what /model_group/info already shows the Admin UI, so a caller budgeting a request does not under-fund the one that lands on the dearest deployment. An unpriced model reports null rather than 0, matching the convention adopted in 40878. The resolved lookup cannot be trusted for this: once the router registers a sparse cost-map entry for a deployment the catalog never priced, the lookup stops raising and defaults the missing costs to 0. The raw cost-map entry is the authority on whether a price exists, so a model the catalog declares at 0.0 still reports 0.0 while one it never priced reports null. Malformed, negative and non-finite values are treated as absent. Resolves #35312 --- litellm/litellm_core_utils/core_helpers.py | 26 ++ litellm/proxy/proxy_server.py | 10 + litellm/proxy/utils.py | 101 ++++++- litellm/router.py | 28 ++ litellm/types/proxy/model_listing.py | 14 +- litellm/types/router.py | 7 + tests/test_litellm/proxy/test_proxy_utils.py | 264 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 16 ++ 8 files changed, 464 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index d29b1fc74ef..c5ff6ed61d0 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal @@ -193,6 +194,31 @@ def coerce_token_limit(value: object) -> int | None: return None +def coerce_token_price(value: object) -> float | None: + """ + Coerce a per-token price to a float, treating a malformed value as absent. + + Prices reach the /v1/models listing from the same uncoerced sources as token limits + (a deployment's model_info is registered into litellm.model_cost verbatim), so a + config value like "0.000003" has to survive while "" does not fail the listing. + + Negative and non-finite values are rejected: a caller pricing a request would read a + negative rate as a discount, and NaN/inf serialize to invalid JSON. Bools are rejected + because True/False is never a meaningful price. + """ + if isinstance(value, bool): + return None + if not isinstance(value, (int, float, str)): + return None + try: + price: Final = float(value) + except (TypeError, ValueError, OverflowError): + return None + if not math.isfinite(price) or price < 0: + return None + return price + + _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = { # Anthropic "stop_sequence": "stop", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7634237a59f..1ae0dc3259b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10963,6 +10963,7 @@ async def model_list( fallback_type: str | None = None, scope: str | None = None, healthy_only: bool | None = False, + include_pricing: bool | None = False, ): """ Use `/model/info` - to get detailed model information, example - pricing, mode, etc. @@ -10971,6 +10972,13 @@ async def model_list( Query Parameters: - include_metadata: Include additional metadata in the response with fallback information + - include_pricing: Include a `pricing` object per model, carrying + `input_cost_per_token` and `output_cost_per_token` in USD per token. + Custom pricing configured on a deployment takes precedence over the + built-in cost map. A price the proxy does not know is reported as + `null`, never 0, so an unmapped model is not read as free. When a + model group's deployments disagree on price, the highest is + reported, matching `/model_group/info`. - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") Defaults to "general" when include_metadata=true - scope: Optional scope parameter. Currently only accepts "expand". @@ -11105,6 +11113,7 @@ async def model_list( include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, + include_pricing=include_pricing or False, ) model_info["id"] = response_id model_data.append(model_info) @@ -11158,6 +11167,7 @@ async def model_list( include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, + include_pricing=include_pricing or False, ) model_info["id"] = response_id model_data.append(model_info) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9de2b5fd282..bf222b11cef 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -70,7 +70,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.proxy.model_listing import ModelInfoResponse +from litellm.types.proxy.model_listing import ModelInfoResponse, ModelPricing from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo, Usage try: @@ -130,6 +130,7 @@ from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_a from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, + coerce_token_price, get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, @@ -253,6 +254,7 @@ if TYPE_CHECKING: from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction from litellm.repositories.prisma_protocols import TableActions from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline + from litellm.types.router import DeploymentModelListingInfo Span = _Span | object else: @@ -8385,6 +8387,89 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: return max(limits) if limits else None +def _declared_price(info: ModelInfo, field: str, cost_map: Mapping[str, object]) -> float | None: + """The price ``field`` carries, but only when the cost map actually declares it. + + A resolved ``ModelInfo`` cannot be trusted on its own here. Once the router registers + a sparse cost-map entry for a deployment the catalog has never priced, the lookup + stops raising and defaults the missing costs to 0, so an unpriced model would be + served as free. The raw entry is the authority on whether a price exists at all, + which is the same test ``_is_cost_explicitly_configured`` makes for budget checks. + + A model the catalog deliberately prices at 0.0 still reports 0.0; one it never priced + reports None. + """ + key: Final = info.get("key") + raw: Final = cost_map.get(key) if isinstance(key, str) else None + if not isinstance(raw, Mapping) or field not in raw: + return None + return coerce_token_price(info.get(field, raw[field])) + + +def _first_price(candidates: tuple[ModelInfo, ...], field: str, cost_map: Mapping[str, object]) -> float | None: + """The price ``field`` takes from the best cost-map entry describing one deployment. + + ``candidates`` is best-source-first (the exact entry for the deployment's underlying + model before a generalized family fallback), so these are not competing prices to + compare: it is one price described with varying reliability, and the first usable + answer wins. + """ + return next( + (price for price in (_declared_price(info, field, cost_map) for info in candidates) if price is not None), + None, + ) + + +def _group_price( + candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: str, cost_map: Mapping[str, object] +) -> float | None: + """The highest price any deployment behind the listed name declares for ``field``. + + Mirrors ``_group_token_limit``: one model behind interchangeable deployments has a + single price and the choice of aggregate does not arise. When a group mixes models, + the highest is reported because it is what ``/model_group/info`` already shows, and + because a caller budgeting against the cheapest member under-funds the request that + lands on the dearest one. + """ + prices: Final = tuple( + price + for price in (_first_price(candidates, field, cost_map) for candidates in candidate_sets) + if price is not None + ) + return max(prices) if prices else None + + +def _listing_pricing( + candidate_sets: tuple[tuple[ModelInfo, ...], ...], + listing_info: "DeploymentModelListingInfo | None", + cost_map: Mapping[str, object], +) -> ModelPricing: + """Effective per-token prices for a listed model, in USD. + + Custom pricing configured on a deployment outranks the cost map, exactly as configured + token limits do, and is the only source for it: the router strips custom pricing from + the shared backend cost-map key so one deployment's override cannot become another's. + + A price the proxy does not know is reported as None rather than 0, so a caller doing + cost accounting cannot read an unmapped model as free. + """ + configured_input: Final = listing_info.input_cost_per_token if listing_info is not None else None + configured_output: Final = listing_info.output_cost_per_token if listing_info is not None else None + pricing: Final[ModelPricing] = { + "input_cost_per_token": ( + configured_input + if configured_input is not None + else _group_price(candidate_sets, "input_cost_per_token", cost_map) + ), + "output_cost_per_token": ( + configured_output + if configured_output is not None + else _group_price(candidate_sets, "output_cost_per_token", cost_map) + ), + } + return pricing + + def create_model_info_response( model_id: str, provider: str, @@ -8392,6 +8477,8 @@ def create_model_info_response( fallback_type: str | None = None, llm_router: Optional["Router"] = None, get_model_info: Callable[[str], ModelInfo] = litellm.get_model_info, + include_pricing: bool = False, + cost_map: Mapping[str, object] | None = None, ) -> ModelInfoResponse: """ Create a standardized OpenAI-compatible model object. @@ -8399,6 +8486,11 @@ def create_model_info_response( When include_metadata is true, attaches the model's configured fallbacks (resolved via the router under fallback_type, defaulting to "general"). Raises HTTPException(400) for an unknown fallback_type. + + When include_pricing is true, attaches the model's effective per-token prices in USD, + resolved from the same deployment-then-cost-map sources as the token limits above. + cost_map defaults to litellm.model_cost, resolved per call because a price reload + rebuilds it. """ from litellm.proxy.auth.model_checks import get_all_fallbacks @@ -8465,6 +8557,13 @@ def create_model_info_response( if max_output_tokens is not None: base["max_output_tokens"] = max_output_tokens + if include_pricing: + base["pricing"] = _listing_pricing( + candidate_sets, + listing_info, + cost_map if cost_map is not None else litellm.model_cost, + ) + if not include_metadata: return base diff --git a/litellm/router.py b/litellm/router.py index 98c7c319eaa..b93edad0b0f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -73,6 +73,7 @@ from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, coerce_token_limit, + coerce_token_price, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, @@ -10263,6 +10264,8 @@ class Router: cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + input_cost_per_token=self._highest_configured_price(model_infos, params, "input_cost_per_token"), + output_cost_per_token=self._highest_configured_price(model_infos, params, "output_cost_per_token"), ) @staticmethod @@ -10275,6 +10278,31 @@ class Router: ) return max(limits) if limits else None + @staticmethod + def _highest_configured_price( + model_infos: Sequence[Mapping[str, Any]], + params: Sequence[Mapping[str, Any]], + field: str, + ) -> float | None: + """The highest custom price for ``field`` across a group's deployments. + + Both sources are read because custom pricing is accepted in ``model_info`` and in + ``litellm_params``; the router itself copies the latter onto the former when it + builds a deployment's cost-map entry. + + The highest is the deliberate pick, matching what ``get_model_group_info`` reports + for a group whose deployments disagree: a caller pricing a request against the + cheapest member would under-budget for the request that lands on the dearest one. + """ + prices: Final = tuple( + price + for price in ( + coerce_token_price(source.get(field)) for sources in (model_infos, params) for source in sources + ) + if price is not None + ) + return max(prices) if prices else None + def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index 24cfa85eee4..6d2c6d0aaa6 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -2,13 +2,24 @@ from typing import Literal -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict class ModelInfoMetadata(TypedDict): fallbacks: list[str] +class ModelPricing(TypedDict): + """Effective token prices for a listed model, in USD per token. + + ``None`` is not the same as free: an unmapped model reports null so a caller doing + cost accounting cannot read a missing price as a zero-cost model. + """ + + input_cost_per_token: ReadOnly[float | None] + output_cost_per_token: ReadOnly[float | None] + + class ModelInfoResponse(TypedDict): """OpenAI-compatible model object. `mode`, `max_input_tokens`, and `max_output_tokens` are attached when the cost map or deployment config @@ -23,3 +34,4 @@ class ModelInfoResponse(TypedDict): max_input_tokens: NotRequired[int] max_output_tokens: NotRequired[int] metadata: NotRequired[ModelInfoMetadata] + pricing: NotRequired[ModelPricing] diff --git a/litellm/types/router.py b/litellm/types/router.py index a75b4654cab..f769e3ec1b7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -642,11 +642,18 @@ class DeploymentModelListingInfo: order, so the ordinary group -- several interchangeable deployments of one model -- carries exactly one. The token limits are the widest explicitly set in any deployment's ``model_info``, which outrank anything the cost map says. + + The token prices are the highest any deployment configures, and the cost map cannot + supply them: a deployment's custom pricing is registered only under its own model id + and is deliberately stripped from the shared backend key, so that one deployment's + override never becomes another's price. """ cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None class RouterErrors(enum.Enum): diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 734408d9b61..4c3291c3022 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1129,6 +1129,270 @@ def test_create_model_info_response_reports_widest_window_in_a_mixed_group(): assert response["max_output_tokens"] == 128000, keys +def test_create_model_info_response_omits_pricing_unless_asked(): + """The default listing shape is unchanged, so existing clients see no new field.""" + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info( + input_cost_per_token=2.5e-06, output_cost_per_token=1e-05 + ), + ) + + assert "pricing" not in response + + +def test_create_model_info_response_includes_cost_map_pricing(): + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05 + ), + cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + } + + +def test_create_model_info_response_reports_unpriced_model_as_null_not_zero(): + """A model the catalog never priced must report null even though the lookup says 0. + + Once the router registers a sparse cost-map entry for an unmapped deployment, the + lookup stops raising and defaults the missing costs to 0. Trusting that value served + an unpriced model as free, which is exactly what #35312 asks the listing not to do. + """ + response = create_model_info_response( + model_id="unknown-model", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="openai/some-model-that-does-not-exist", + input_cost_per_token=0, + output_cost_per_token=0, + ), + cost_map={"openai/some-model-that-does-not-exist": {"id": "deployment-hash"}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": None, + "output_cost_per_token": None, + } + + +def test_create_model_info_response_reports_declared_zero_price_as_zero(): + """A model the catalog deliberately prices at 0 is free, and must not report null. + + This is the counterpart to the unpriced case: both resolve to 0, only the raw cost-map + entry distinguishes "free" from "nobody ever priced this". + """ + response = create_model_info_response( + model_id="free-model", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="free-model", input_cost_per_token=0.0, output_cost_per_token=0.0 + ), + cost_map={"free-model": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + + +def test_create_model_info_response_configured_pricing_overrides_cost_map(): + """Custom pricing is the price the request is billed at, so it outranks the cost map. + + The cost map cannot carry it: the router strips custom pricing from the shared backend + key so one deployment's override never becomes another's price. + """ + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("gpt-4o",), + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=3e-07, + output_cost_per_token=1.2e-06, + ) + + response = create_model_info_response( + model_id="house-gpt", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05 + ), + cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + } + + +def test_create_model_info_response_reports_highest_price_in_a_mixed_group(): + """A group mixing models quotes the dearest member, matching /model_group/info. + + Quoting the cheapest would under-budget every request that lands on the dearest + deployment, and the listing cannot know which one a later request will reach. + """ + prices = { + "cheap-model": _fake_model_info( + key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07 + ), + "dear-model": _fake_model_info( + key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05 + ), + } + cost_map = { + "cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07}, + "dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05}, + } + + for keys in (("cheap-model", "dear-model"), ("dear-model", "cheap-model")): + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=keys, max_input_tokens=None, max_output_tokens=None + ) + + response = create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda model: prices[model], + cost_map=cost_map, + ) + + assert response["pricing"] == { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + }, keys + + +def test_create_model_info_response_reports_unknown_price_as_null(): + """An unmapped model must not read as free to a caller doing cost accounting.""" + response = create_model_info_response( + model_id="my-custom-deployment", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=_raise_unmapped, + ) + + assert response["pricing"] == { + "input_cost_per_token": None, + "output_cost_per_token": None, + } + + +def test_create_model_info_response_ignores_malformed_price(): + """A malformed or negative price is dropped rather than served or blowing up the listing. + + model_info is registered into the cost map verbatim, so a config typo reaches here + uncoerced; a negative rate would read as a discount and NaN is not valid JSON. + """ + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token="", output_cost_per_token=-1e-06 + ), + cost_map={"gpt-4o": {"input_cost_per_token": "", "output_cost_per_token": -1e-06}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": None, + "output_cost_per_token": None, + } + + +def test_create_model_info_response_accepts_string_price(): + """A price configured as a string still reaches the listing as a number.""" + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=None, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token="0.0000025", output_cost_per_token="0.00001" + ), + cost_map={"gpt-4o": {"input_cost_per_token": "0.0000025", "output_cost_per_token": "0.00001"}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + } + + +def test_create_model_info_response_pricing_does_not_call_router_group_info(): + """Pricing must not pull /v1/models onto the expensive group-info path (#33721).""" + router = MagicMock() + router.get_model_listing_info.return_value = None + + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info(key="gpt-4o", input_cost_per_token=2.5e-06), + cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06}}, + ) + + router.get_model_group_info.assert_not_called() + assert response["pricing"]["input_cost_per_token"] == 2.5e-06 + + +def test_get_model_listing_info_carries_custom_deployment_pricing(): + """Custom pricing set in litellm_params reaches the listing through the router index.""" + router = litellm.Router( + model_list=[ + { + "model_name": "house-gpt", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + }, + }, + ] + ) + + listing_info = router.get_model_listing_info("house-gpt") + + assert listing_info is not None + assert listing_info.input_cost_per_token == 3e-07 + assert listing_info.output_cost_per_token == 1.2e-06 + + response = create_model_info_response( + model_id="house-gpt", + provider="openai", + llm_router=router, + include_pricing=True, + ) + + assert response["pricing"] == { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + } + + def test_create_model_info_response_resolves_alias_once_per_listing(): """The alias is the same for every deployment in the group, so it is looked up once.""" seen: list[str] = [] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e33764c3d1a..36429cb8857 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9678,6 +9678,13 @@ export interface paths { * * Query Parameters: * - include_metadata: Include additional metadata in the response with fallback information + * - include_pricing: Include a `pricing` object per model, carrying + * `input_cost_per_token` and `output_cost_per_token` in USD per token. + * Custom pricing configured on a deployment takes precedence over the + * built-in cost map. A price the proxy does not know is reported as + * `null`, never 0, so an unmapped model is not read as free. When a + * model group's deployments disagree on price, the highest is + * reported, matching `/model_group/info`. * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true * - scope: Optional scope parameter. Currently only accepts "expand". @@ -19666,6 +19673,13 @@ export interface paths { * * Query Parameters: * - include_metadata: Include additional metadata in the response with fallback information + * - include_pricing: Include a `pricing` object per model, carrying + * `input_cost_per_token` and `output_cost_per_token` in USD per token. + * Custom pricing configured on a deployment takes precedence over the + * built-in cost map. A price the proxy does not know is reported as + * `null`, never 0, so an unmapped model is not read as free. When a + * model group's deployments disagree on price, the highest is + * reported, matching `/model_group/info`. * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true * - scope: Optional scope parameter. Currently only accepts "expand". @@ -54527,6 +54541,7 @@ export interface operations { fallback_type?: string | null; scope?: string | null; healthy_only?: boolean | null; + include_pricing?: boolean | null; }; header?: never; path?: never; @@ -67211,6 +67226,7 @@ export interface operations { fallback_type?: string | null; scope?: string | null; healthy_only?: boolean | null; + include_pricing?: boolean | null; }; header?: never; path?: never; From d9676f8c2a5b379c6fbf322a4b244f76e20c375b Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Mon, 21 Sep 2026 10:04:37 +0530 Subject: [PATCH 2/9] test(router): cover _highest_configured_price directly The listing tests exercise this helper through get_model_listing_info, which leaves it invisible to tests/code_coverage_tests/router_code_coverage.py: that gate only counts calls made from test files whose name contains "router". Covers what the helper guarantees: custom pricing is read from both model_info and litellm_params, a group whose deployments disagree quotes the dearest, an unconfigured group reports None rather than 0, and a malformed or negative configured value is dropped. --- tests/test_litellm/test_router.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 8310d30d90e..e25f8ea716a 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9539,6 +9539,41 @@ def test_get_model_listing_info_returns_none_for_unknown_name(): assert router.get_model_listing_info("not-a-real-model") is None +def test_highest_configured_price_quotes_the_dearest_deployment_across_both_sources(): + """Custom pricing is accepted in model_info and in litellm_params, so both are read. + + A group whose deployments disagree quotes the dearest, matching what + get_model_group_info reports: a caller pricing against the cheapest member + under-funds the request that lands on the dearest one. + """ + model_infos = ({"input_cost_per_token": 3e-07}, {"mode": "chat"}) + params = ({"model": "openai/gpt-4o-mini"}, {"input_cost_per_token": 9e-07}) + + highest = litellm.Router._highest_configured_price(model_infos, params, "input_cost_per_token") + + assert highest == 9e-07, f"expected the dearer deployment's price, got {highest}" + + +def test_highest_configured_price_is_none_when_no_deployment_configures_one(): + """Nothing configured means the cost map answers instead, so this must not report 0.""" + model_infos = ({"mode": "chat"},) + params = ({"model": "openai/gpt-4o-mini"},) + + highest = litellm.Router._highest_configured_price(model_infos, params, "input_cost_per_token") + + assert highest is None, f"expected no configured price, got {highest}" + + +def test_highest_configured_price_ignores_a_malformed_configured_value(): + """model_info reaches the listing uncoerced, so a config typo must not become a price.""" + model_infos = ({"input_cost_per_token": ""},) + params = ({"model": "openai/gpt-4o-mini", "input_cost_per_token": -1e-06},) + + highest = litellm.Router._highest_configured_price(model_infos, params, "input_cost_per_token") + + assert highest is None, f"expected malformed and negative values to be dropped, got {highest}" + + def test_get_model_listing_info_carries_configured_limits(): router = litellm.Router( model_list=[ From 215b4583587d9d6dd50051920b3e74891f5c4d33 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Mon, 21 Sep 2026 12:55:58 +0530 Subject: [PATCH 3/9] fix(proxy): price each deployment before the listing compares them Review found two configurations where /v1/models advertised less than the request would be billed, and less than /model_group/info reports. A group mixing an overridden deployment with an uncustomised one quoted the override: prices were compared among overrides, and the cost map was consulted only when no deployment configured anything. A group of gpt-4o-mini at 3e-07 and gpt-5.5 at its catalog 5e-06 therefore advertised 3e-07, roughly 17x under, while every request routed to the second deployment billed 5e-06. Each deployment now resolves its own effective price first, its override when it has one and its own cost-map entry otherwise, and the group reports the highest of those. A single deployment whose override is cheaper than its catalog price still reports the override, because that is what the request is billed at. DeploymentModelListingInfo carries one DeploymentListingPrice per deployment instead of two group-wide maxima. Wildcard-expanded rows had no record at all: the deployment is indexed under its pattern, so `openai/*` priced at 9e-06 listed its expanded rows at the catalog's 1.5e-07, 60x under. Those rows now ask the pattern router for the deployment behind them, behind two guards, so ordinary rows and configs without a priced wildcard pay nothing: only names the index misses reach it, and only when some wildcard deployment actually configures a price. get_model_group_info is still never called, so the listing stays off the expensive path. --- litellm/proxy/utils.py | 107 +++++++++---- litellm/router.py | 78 +++++++--- litellm/types/router.py | 28 +++- tests/test_litellm/proxy/test_proxy_utils.py | 152 ++++++++++++++++++- tests/test_litellm/test_router.py | 121 ++++++++++++--- 5 files changed, 403 insertions(+), 83 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bf222b11cef..b10420e9d50 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -71,6 +71,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse, ModelPricing +from litellm.types.router import DeploymentListingPrice from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo, Usage try: @@ -8420,51 +8421,98 @@ def _first_price(candidates: tuple[ModelInfo, ...], field: str, cost_map: Mappin ) -def _group_price( - candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: str, cost_map: Mapping[str, object] -) -> float | None: - """The highest price any deployment behind the listed name declares for ``field``. +def _listing_deployment_prices( + lookup_model: str, + listing_info: "DeploymentModelListingInfo | None", + llm_router: Optional["Router"], +) -> tuple["DeploymentListingPrice", ...]: + """The per-deployment prices behind a listed name, one entry per deployment. - Mirrors ``_group_token_limit``: one model behind interchangeable deployments has a - single price and the choice of aggregate does not arise. When a group mixes models, - the highest is reported because it is what ``/model_group/info`` already shows, and - because a caller budgeting against the cheapest member under-funds the request that - lands on the dearest one. + A name the router indexes carries its deployments' own records. A wildcard-expanded + name has none, so its deployment's override lives on the pattern instead and is asked + for there; the record is keyed to None so an absent half still falls back to the + catalog entry the listing already resolved. """ - prices: Final = tuple( - price - for price in (_first_price(candidates, field, cost_map) for candidates in candidate_sets) - if price is not None + if listing_info is not None: + return listing_info.deployment_prices + + wildcard_price: Final = llm_router.get_wildcard_listing_price(lookup_model) if llm_router is not None else None + if wildcard_price is None: + return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),) + return ( + DeploymentListingPrice( + cost_map_key=None, + input_cost_per_token=wildcard_price.input_cost_per_token, + output_cost_per_token=wildcard_price.output_cost_per_token, + ), ) - return max(prices) if prices else None + + +def _deployment_effective_price( + configured: float | None, + cost_map_key: str | None, + field: str, + candidates_by_key: Mapping[str | None, tuple[ModelInfo, ...]], + cost_map: Mapping[str, object], +) -> float | None: + """What one deployment charges for ``field``: its own override, else its catalog entry.""" + if configured is not None: + return configured + return _first_price(candidates_by_key.get(cost_map_key, ()), field, cost_map) + + +def _highest_price(prices: tuple[float | None, ...]) -> float | None: + known: Final = tuple(price for price in prices if price is not None) + return max(known) if known else None def _listing_pricing( candidate_sets: tuple[tuple[ModelInfo, ...], ...], - listing_info: "DeploymentModelListingInfo | None", + deployment_models: tuple[str | None, ...], + deployment_prices: tuple["DeploymentListingPrice", ...], cost_map: Mapping[str, object], ) -> ModelPricing: """Effective per-token prices for a listed model, in USD. - Custom pricing configured on a deployment outranks the cost map, exactly as configured - token limits do, and is the only source for it: the router strips custom pricing from - the shared backend cost-map key so one deployment's override cannot become another's. + Each deployment resolves its own price first, its override when it has one and its + catalog entry otherwise, and the group reports the highest of those. Comparing + overrides against each other and only then falling back to the catalog would hide a + dearer deployment that happens to configure no override, quoting a caller less than + the request that lands there actually costs. + + The highest is the deliberate pick across a group, matching what ``/model_group/info`` + reports, because the listing cannot know which deployment a later request will reach. A price the proxy does not know is reported as None rather than 0, so a caller doing cost accounting cannot read an unmapped model as free. """ - configured_input: Final = listing_info.input_cost_per_token if listing_info is not None else None - configured_output: Final = listing_info.output_cost_per_token if listing_info is not None else None + candidates_by_key: Final[Mapping[str | None, tuple[ModelInfo, ...]]] = MappingProxyType( + {key: candidates for key, candidates in zip(deployment_models, candidate_sets)} + ) pricing: Final[ModelPricing] = { - "input_cost_per_token": ( - configured_input - if configured_input is not None - else _group_price(candidate_sets, "input_cost_per_token", cost_map) + "input_cost_per_token": _highest_price( + tuple( + _deployment_effective_price( + deployment.input_cost_per_token, + deployment.cost_map_key, + "input_cost_per_token", + candidates_by_key, + cost_map, + ) + for deployment in deployment_prices + ) ), - "output_cost_per_token": ( - configured_output - if configured_output is not None - else _group_price(candidate_sets, "output_cost_per_token", cost_map) + "output_cost_per_token": _highest_price( + tuple( + _deployment_effective_price( + deployment.output_cost_per_token, + deployment.cost_map_key, + "output_cost_per_token", + candidates_by_key, + cost_map, + ) + for deployment in deployment_prices + ) ), } return pricing @@ -8560,7 +8608,8 @@ def create_model_info_response( if include_pricing: base["pricing"] = _listing_pricing( candidate_sets, - listing_info, + deployment_models, + _listing_deployment_prices(lookup_model, listing_info, llm_router), cost_map if cost_map is not None else litellm.model_cost, ) diff --git a/litellm/router.py b/litellm/router.py index b93edad0b0f..866098baf2f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -251,6 +251,7 @@ from litellm.types.router import ( CredentialLiteLLMParams, CustomRoutingStrategyBase, Deployment, + DeploymentListingPrice, DeploymentModelListingInfo, DeploymentTypedDict, DiscoveredDeploymentModelInfo, @@ -10264,8 +10265,10 @@ class Router: cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), - input_cost_per_token=self._highest_configured_price(model_infos, params, "input_cost_per_token"), - output_cost_per_token=self._highest_configured_price(model_infos, params, "output_cost_per_token"), + deployment_prices=tuple( + self._deployment_listing_price(model_info, litellm_params) + for model_info, litellm_params in zip(model_infos, params) + ), ) @staticmethod @@ -10279,29 +10282,68 @@ class Router: return max(limits) if limits else None @staticmethod - def _highest_configured_price( - model_infos: Sequence[Mapping[str, Any]], - params: Sequence[Mapping[str, Any]], + def _configured_price( + model_info: Mapping[str, Any], + litellm_params: Mapping[str, Any], field: str, ) -> float | None: - """The highest custom price for ``field`` across a group's deployments. + """One deployment's custom price for ``field``, or None when it configures none. Both sources are read because custom pricing is accepted in ``model_info`` and in ``litellm_params``; the router itself copies the latter onto the former when it - builds a deployment's cost-map entry. - - The highest is the deliberate pick, matching what ``get_model_group_info`` reports - for a group whose deployments disagree: a caller pricing a request against the - cheapest member would under-budget for the request that lands on the dearest one. + builds a deployment's cost-map entry. ``litellm_params`` wins, matching that copy. """ - prices: Final = tuple( - price - for price in ( - coerce_token_price(source.get(field)) for sources in (model_infos, params) for source in sources - ) - if price is not None + from_params: Final = coerce_token_price(litellm_params.get(field)) + return from_params if from_params is not None else coerce_token_price(model_info.get(field)) + + @staticmethod + def _deployment_listing_price( + model_info: Mapping[str, Any], + litellm_params: Mapping[str, Any], + ) -> DeploymentListingPrice: + """What one deployment contributes to its listed name's price.""" + cost_map_key: Final = ( + model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") ) - return max(prices) if prices else None + return DeploymentListingPrice( + cost_map_key=cost_map_key if isinstance(cost_map_key, str) and cost_map_key else None, + input_cost_per_token=Router._configured_price(model_info, litellm_params, "input_cost_per_token"), + output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"), + ) + + def get_wildcard_listing_price(self, model_name: str) -> DeploymentListingPrice | None: + """Custom pricing a wildcard deployment configures for a name it expands to. + + Wildcard-expanded rows are absent from the model-name index, so their deployment's + override is invisible to ``get_model_listing_info`` and the listing would quote the + catalog while the request is billed at the override. + + Pattern matching is the expensive path this listing exists to avoid, so it only + runs when a wildcard deployment actually configures a price: with none, the catalog + is already the right answer and nothing here can change it. + """ + if not any( + Router._deployment_has_configured_price(deployment) + for deployments in self.pattern_router.patterns.values() + for deployment in deployments + ): + return None + + matched: Final = self.pattern_router.route(model_name) + if not matched: + return None + return Router._deployment_listing_price( + matched[0].get("model_info") or MappingProxyType({}), + matched[0].get("litellm_params") or MappingProxyType({}), + ) + + @staticmethod + def _deployment_has_configured_price(deployment: Mapping[str, Any]) -> bool: + price: Final = Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + return price.input_cost_per_token is not None or price.output_cost_per_token is not None def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ diff --git a/litellm/types/router.py b/litellm/types/router.py index f769e3ec1b7..2e767c19513 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -631,6 +631,22 @@ class DiscoveredDeploymentModelInfo: limits: Mapping[str, int] +@dataclass(frozen=True, slots=True) +class DeploymentListingPrice: + """What one deployment behind a listed name contributes to its price. + + ``cost_map_key`` names the catalog entry that deployment resolves to, so a caller can + fall back to the catalog for a deployment that configures no price of its own. The + prices are whatever that single deployment configures, never a group aggregate: a + group's price is only correct once each member resolves its own effective price + first, since a deployment without an override can still be the dearest one. + """ + + cost_map_key: str | None + input_cost_per_token: float | None + output_cost_per_token: float | None + + @dataclass(frozen=True, slots=True) class DeploymentModelListingInfo: """What the deployments behind a model name contribute to its OpenAI-compatible listing entry. @@ -643,17 +659,17 @@ class DeploymentModelListingInfo: carries exactly one. The token limits are the widest explicitly set in any deployment's ``model_info``, which outrank anything the cost map says. - The token prices are the highest any deployment configures, and the cost map cannot - supply them: a deployment's custom pricing is registered only under its own model id - and is deliberately stripped from the shared backend key, so that one deployment's - override never becomes another's price. + ``deployment_prices`` carries one entry per deployment, in config order, rather than a + group aggregate: the cost map cannot supply custom pricing on its own, because a + deployment's override is registered only under its own model id and is deliberately + stripped from the shared backend key so one deployment's override never becomes + another's price. """ cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None - input_cost_per_token: float | None = None - output_cost_per_token: float | None = None + deployment_prices: tuple[DeploymentListingPrice, ...] = () class RouterErrors(enum.Enum): diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 4c3291c3022..0fcb8182962 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -988,7 +988,7 @@ from typing import cast import litellm from litellm.proxy.utils import create_model_info_response -from litellm.types.router import DeploymentModelListingInfo +from litellm.types.router import DeploymentListingPrice, DeploymentModelListingInfo from litellm.types.utils import ModelInfo @@ -1221,8 +1221,11 @@ def test_create_model_info_response_configured_pricing_overrides_cost_map(): cost_map_keys=("gpt-4o",), max_input_tokens=None, max_output_tokens=None, - input_cost_per_token=3e-07, - output_cost_per_token=1.2e-06, + deployment_prices=( + DeploymentListingPrice( + cost_map_key="gpt-4o", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06 + ), + ), ) response = create_model_info_response( @@ -1264,7 +1267,13 @@ def test_create_model_info_response_reports_highest_price_in_a_mixed_group(): for keys in (("cheap-model", "dear-model"), ("dear-model", "cheap-model")): router = MagicMock() router.get_model_listing_info.return_value = DeploymentModelListingInfo( - cost_map_keys=keys, max_input_tokens=None, max_output_tokens=None + cost_map_keys=keys, + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=tuple( + DeploymentListingPrice(cost_map_key=key, input_cost_per_token=None, output_cost_per_token=None) + for key in keys + ), ) response = create_model_info_response( @@ -1282,6 +1291,133 @@ def test_create_model_info_response_reports_highest_price_in_a_mixed_group(): }, keys +def test_create_model_info_response_does_not_let_a_custom_rate_hide_a_dearer_catalog_one(): + """One deployment's override must not stand in for a group that has a dearer member. + + Comparing overrides against each other and only then falling back to the catalog + quoted the cheap override here while a request routed to the uncustomised deployment + billed 17x that, and disagreed with /model_group/info. + """ + prices = { + "cheap-model": _fake_model_info(key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07), + "dear-model": _fake_model_info(key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05), + } + cost_map = { + "cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07}, + "dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05}, + } + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("cheap-model", "dear-model"), + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=( + DeploymentListingPrice( + cost_map_key="cheap-model", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06 + ), + DeploymentListingPrice(cost_map_key="dear-model", input_cost_per_token=None, output_cost_per_token=None), + ), + ) + + response = create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda model: prices[model], + cost_map=cost_map, + ) + + assert response["pricing"] == { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + } + + +def test_create_model_info_response_prefers_a_cheaper_override_over_its_own_catalog_price(): + """The override is what the request is billed at, so it wins even when it is cheaper. + + The guard against the case above must not turn into "always take the catalog". + """ + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("gpt-4o",), + max_input_tokens=None, + max_output_tokens=None, + deployment_prices=( + DeploymentListingPrice(cost_map_key="gpt-4o", input_cost_per_token=3e-08, output_cost_per_token=4e-08), + ), + ) + + response = create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 3e-08, + "output_cost_per_token": 4e-08, + } + + +def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): + """A wildcard-expanded name has no index entry, so its override lives on the pattern. + + Reading only the index quoted the catalog here while the request billed at the + wildcard's own rate. + """ + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_price.return_value = DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9e-05, + } + + +def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row(): + """An unpriced wildcard changes nothing, so the catalog answer still stands.""" + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_price.return_value = None + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + } + + def test_create_model_info_response_reports_unknown_price_as_null(): """An unmapped model must not read as free to a caller doing cost accounting.""" response = create_model_info_response( @@ -1344,6 +1480,7 @@ def test_create_model_info_response_pricing_does_not_call_router_group_info(): """Pricing must not pull /v1/models onto the expensive group-info path (#33721).""" router = MagicMock() router.get_model_listing_info.return_value = None + router.get_wildcard_listing_price.return_value = None response = create_model_info_response( model_id="gpt-4o", @@ -1377,8 +1514,11 @@ def test_get_model_listing_info_carries_custom_deployment_pricing(): listing_info = router.get_model_listing_info("house-gpt") assert listing_info is not None - assert listing_info.input_cost_per_token == 3e-07 - assert listing_info.output_cost_per_token == 1.2e-06 + assert listing_info.deployment_prices == ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06 + ), + ) response = create_model_info_response( model_id="house-gpt", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index e25f8ea716a..63b33638460 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -51,7 +51,15 @@ from litellm.router_utils.client_initalization_utils import MaxParallelRequestsL from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.llms.openai import ChatCompletionRequest -from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy +from litellm.types.router import ( + Deployment, + DeploymentListingPrice, + DeploymentTypedDict, + LiteLLM_Params, + ModelInfo, + PreRoutingHookResponse, + RetryPolicy, +) def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): @@ -9539,39 +9547,104 @@ def test_get_model_listing_info_returns_none_for_unknown_name(): assert router.get_model_listing_info("not-a-real-model") is None -def test_highest_configured_price_quotes_the_dearest_deployment_across_both_sources(): - """Custom pricing is accepted in model_info and in litellm_params, so both are read. +def test_configured_price_reads_model_info_when_litellm_params_has_none(): + """Custom pricing is accepted in both blocks, so model_info alone still counts.""" + price = litellm.Router._configured_price( + {"input_cost_per_token": 3e-07}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token" + ) - A group whose deployments disagree quotes the dearest, matching what - get_model_group_info reports: a caller pricing against the cheapest member - under-funds the request that lands on the dearest one. - """ - model_infos = ({"input_cost_per_token": 3e-07}, {"mode": "chat"}) - params = ({"model": "openai/gpt-4o-mini"}, {"input_cost_per_token": 9e-07}) - - highest = litellm.Router._highest_configured_price(model_infos, params, "input_cost_per_token") - - assert highest == 9e-07, f"expected the dearer deployment's price, got {highest}" + assert price == 3e-07, f"expected the model_info price, got {price}" -def test_highest_configured_price_is_none_when_no_deployment_configures_one(): +def test_configured_price_is_none_when_the_deployment_configures_none(): """Nothing configured means the cost map answers instead, so this must not report 0.""" - model_infos = ({"mode": "chat"},) - params = ({"model": "openai/gpt-4o-mini"},) + price = litellm.Router._configured_price( + {"mode": "chat"}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token" + ) - highest = litellm.Router._highest_configured_price(model_infos, params, "input_cost_per_token") - - assert highest is None, f"expected no configured price, got {highest}" + assert price is None, f"expected no configured price, got {price}" -def test_highest_configured_price_ignores_a_malformed_configured_value(): +def test_configured_price_ignores_a_malformed_configured_value(): """model_info reaches the listing uncoerced, so a config typo must not become a price.""" - model_infos = ({"input_cost_per_token": ""},) - params = ({"model": "openai/gpt-4o-mini", "input_cost_per_token": -1e-06},) + price = litellm.Router._configured_price( + {"input_cost_per_token": ""}, {"input_cost_per_token": -1e-06}, "input_cost_per_token" + ) - highest = litellm.Router._highest_configured_price(model_infos, params, "input_cost_per_token") + assert price is None, f"expected malformed and negative values to be dropped, got {price}" - assert highest is None, f"expected malformed and negative values to be dropped, got {highest}" + +def test_get_model_listing_info_keeps_each_deployment_price_separate(): + """A group's price is only correct once each deployment resolves its own price first. + + Collapsing the group to the highest override here would hide a deployment that + configures nothing and is dearer in the catalog, so the listing gets one record per + deployment, carrying that deployment's own cost-map key. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "input_cost_per_token": 3e-07, + }, + }, + {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "test-key"}}, + ] + ) + + info = router.get_model_listing_info("mixed") + + assert info is not None + assert info.deployment_prices == ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=None + ), + DeploymentListingPrice(cost_map_key="openai/gpt-5.5", input_cost_per_token=None, output_cost_per_token=None), + ) + + +def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment(): + """A wildcard-expanded name is absent from the index, so its override lives on the pattern. + + Without this the listing quotes the catalog while the request is billed at the + override. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "test-key", + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9e-05, + }, + } + ] + ) + + price = router.get_wildcard_listing_price("openai/gpt-4o-mini") + + assert price is not None + assert (price.input_cost_per_token, price.output_cost_per_token) == (9e-06, 9e-05) + + +def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_priced(): + """Pattern matching is the expensive path, so an unpriced wildcard must not trigger it. + + With no override the catalog is already the right answer, so there is nothing to find. + """ + router = litellm.Router( + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}}] + ) + + with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed: + assert router.get_wildcard_listing_price("openai/gpt-4o-mini") is None + + routed.assert_not_called() def test_get_model_listing_info_carries_configured_limits(): From d8d90185525fe0f345dd75e99f75c7917188437e Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Mon, 21 Sep 2026 14:22:51 +0530 Subject: [PATCH 4/9] fix(proxy): report the dearest deployment behind a wildcard pattern A pattern can front several deployments and routing picks among them, but the listing read only the first match, so `openai/*` deployments priced 9e-06 and 4e-05 advertised 9e-06 while /model_group/info reported 4e-05. get_wildcard_listing_price becomes get_wildcard_listing_prices and returns one record per matched deployment. Each then resolves its own effective price and the group reports the highest, the same aggregation an indexed group already gets, so a matched deployment configuring nothing is billed at its catalog rate and can be the dearest member: a 1e-08 override beside an uncustomised sibling now reports the catalog's 1.5e-07 rather than 1e-08. The guards are unchanged: only names the model-name index misses reach the pattern router, and only when some wildcard deployment configures a price. Also drops _deployment_has_configured_price, inlined into its only caller, and covers _deployment_listing_price directly: tests/code_coverage_tests/ router_code_coverage.py counts only calls made by name from a test file whose name contains "router". --- litellm/proxy/utils.py | 20 +++--- litellm/router.py | 48 +++++++------ tests/test_litellm/proxy/test_proxy_utils.py | 73 ++++++++++++++++++-- tests/test_litellm/test_router.py | 49 +++++++++---- 4 files changed, 142 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b10420e9d50..80ea126343c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8429,22 +8429,24 @@ def _listing_deployment_prices( """The per-deployment prices behind a listed name, one entry per deployment. A name the router indexes carries its deployments' own records. A wildcard-expanded - name has none, so its deployment's override lives on the pattern instead and is asked - for there; the record is keyed to None so an absent half still falls back to the - catalog entry the listing already resolved. + name has none, so the overrides live on the patterns it expands from and are asked for + there, one record per matched deployment because a request can route to any of them. + Those records are keyed to None so a deployment that configures nothing, or configures + only one half, still falls back to the catalog entry the listing already resolved. """ if listing_info is not None: return listing_info.deployment_prices - wildcard_price: Final = llm_router.get_wildcard_listing_price(lookup_model) if llm_router is not None else None - if wildcard_price is None: + wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else () + if not wildcard_prices: return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),) - return ( + return tuple( DeploymentListingPrice( cost_map_key=None, - input_cost_per_token=wildcard_price.input_cost_per_token, - output_cost_per_token=wildcard_price.output_cost_per_token, - ), + input_cost_per_token=price.input_cost_per_token, + output_cost_per_token=price.output_cost_per_token, + ) + for price in wildcard_prices ) diff --git a/litellm/router.py b/litellm/router.py index 866098baf2f..bca31b6427f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10311,40 +10311,42 @@ class Router: output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"), ) - def get_wildcard_listing_price(self, model_name: str) -> DeploymentListingPrice | None: - """Custom pricing a wildcard deployment configures for a name it expands to. + def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]: + """One record per wildcard deployment that a name expands from, in match order. - Wildcard-expanded rows are absent from the model-name index, so their deployment's - override is invisible to ``get_model_listing_info`` and the listing would quote the - catalog while the request is billed at the override. + Wildcard-expanded rows are absent from the model-name index, so their deployments' + overrides are invisible to ``get_model_listing_info`` and the listing would quote + the catalog while the request is billed at an override. + + Every matched deployment is returned rather than the first, because a request can + route to any of them: the caller resolves each one and reports the dearest, the same + way it treats a group the index does know. Pattern matching is the expensive path this listing exists to avoid, so it only runs when a wildcard deployment actually configures a price: with none, the catalog is already the right answer and nothing here can change it. """ if not any( - Router._deployment_has_configured_price(deployment) - for deployments in self.pattern_router.patterns.values() - for deployment in deployments + price.input_cost_per_token is not None or price.output_cost_per_token is not None + for price in ( + Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + for deployments in self.pattern_router.patterns.values() + for deployment in deployments + ) ): - return None + return () - matched: Final = self.pattern_router.route(model_name) - if not matched: - return None - return Router._deployment_listing_price( - matched[0].get("model_info") or MappingProxyType({}), - matched[0].get("litellm_params") or MappingProxyType({}), + return tuple( + Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + for deployment in (self.pattern_router.route(model_name) or ()) ) - @staticmethod - def _deployment_has_configured_price(deployment: Mapping[str, Any]) -> bool: - price: Final = Router._deployment_listing_price( - deployment.get("model_info") or MappingProxyType({}), - deployment.get("litellm_params") or MappingProxyType({}), - ) - return price.input_cost_per_token is not None or price.output_cost_per_token is not None - def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 0fcb8182962..dfab3d9cfac 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1374,8 +1374,10 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): """ router = MagicMock() router.get_model_listing_info.return_value = None - router.get_wildcard_listing_price.return_value = DeploymentListingPrice( - cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ), ) response = create_model_info_response( @@ -1395,11 +1397,74 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): } +def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern(): + """A pattern can front several deployments, and routing can pick any of them. + + Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing + advertised less than the request could cost. + """ + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ), + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05 + ), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 4e-05, + "output_cost_per_token": 9e-05, + } + + +def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog(): + """A matched deployment configuring nothing is billed at the catalog, so it can be the + dearest member even when its sibling carries a very cheap override.""" + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08 + ), + DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + } + + def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row(): """An unpriced wildcard changes nothing, so the catalog answer still stands.""" router = MagicMock() router.get_model_listing_info.return_value = None - router.get_wildcard_listing_price.return_value = None + router.get_wildcard_listing_prices.return_value = () response = create_model_info_response( model_id="openai/gpt-4o-mini", @@ -1480,7 +1545,7 @@ def test_create_model_info_response_pricing_does_not_call_router_group_info(): """Pricing must not pull /v1/models onto the expensive group-info path (#33721).""" router = MagicMock() router.get_model_listing_info.return_value = None - router.get_wildcard_listing_price.return_value = None + router.get_wildcard_listing_prices.return_value = () response = create_model_info_response( model_id="gpt-4o", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 63b33638460..b2f81c31d56 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9574,6 +9574,23 @@ def test_configured_price_ignores_a_malformed_configured_value(): assert price is None, f"expected malformed and negative values to be dropped, got {price}" +def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params(): + """The record a deployment contributes: its catalog key, plus its own configured prices. + + base_model names the catalog entry when set, because litellm_params.model can be an + opaque backend name, and litellm_params wins over model_info for a price, matching the + copy the router makes when it builds the deployment's cost-map entry. + """ + price = litellm.Router._deployment_listing_price( + {"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06}, + {"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06}, + ) + + assert price == DeploymentListingPrice( + cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None + ) + + def test_get_model_listing_info_keeps_each_deployment_price_separate(): """A group's price is only correct once each deployment resolves its own price first. @@ -9606,12 +9623,9 @@ def test_get_model_listing_info_keeps_each_deployment_price_separate(): ) -def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment(): - """A wildcard-expanded name is absent from the index, so its override lives on the pattern. - - Without this the listing quotes the catalog while the request is billed at the - override. - """ +def test_get_wildcard_listing_prices_reports_every_matched_deployment(): + """A pattern can front several deployments and routing can pick any of them, so the + listing needs all their prices, not the first one's: the caller reports the dearest.""" router = litellm.Router( model_list=[ { @@ -9622,17 +9636,28 @@ def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment(): "input_cost_per_token": 9e-06, "output_cost_per_token": 9e-05, }, - } + }, + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "test-key", + "input_cost_per_token": 4e-05, + "output_cost_per_token": 8e-05, + }, + }, ] ) - price = router.get_wildcard_listing_price("openai/gpt-4o-mini") + prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini") - assert price is not None - assert (price.input_cost_per_token, price.output_cost_per_token) == (9e-06, 9e-05) + assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == { + (9e-06, 9e-05), + (4e-05, 8e-05), + } -def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_priced(): +def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced(): """Pattern matching is the expensive path, so an unpriced wildcard must not trigger it. With no override the catalog is already the right answer, so there is nothing to find. @@ -9642,7 +9667,7 @@ def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_p ) with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed: - assert router.get_wildcard_listing_price("openai/gpt-4o-mini") is None + assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == () routed.assert_not_called() From 2909f1d7e3471f41f356603d3aa03cf4929ff362 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Mon, 21 Sep 2026 14:38:21 +0530 Subject: [PATCH 5/9] test(pricing): carry the expectation in the assertion message tests/AGENTS.md asks for context in the assert message rather than around it, so a failure reads on its own: the pricing tests explained themselves in docstrings and then asserted bare equality. Every pricing assertion now names what it expected and prints what it got. The docstrings stay, because they carry the reason the rule exists rather than the expectation itself. --- tests/test_litellm/proxy/test_proxy_utils.py | 30 ++++++++++---------- tests/test_litellm/test_router.py | 6 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index dfab3d9cfac..9d3421cadc1 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1158,7 +1158,7 @@ def test_create_model_info_response_includes_cost_map_pricing(): assert response["pricing"] == { "input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05, - } + }, f"expected the cost-map prices, got {response['pricing']}" def test_create_model_info_response_reports_unpriced_model_as_null_not_zero(): @@ -1184,7 +1184,7 @@ def test_create_model_info_response_reports_unpriced_model_as_null_not_zero(): assert response["pricing"] == { "input_cost_per_token": None, "output_cost_per_token": None, - } + }, f"a sparse cost-map entry must read as unknown, got {response['pricing']}" def test_create_model_info_response_reports_declared_zero_price_as_zero(): @@ -1207,7 +1207,7 @@ def test_create_model_info_response_reports_declared_zero_price_as_zero(): assert response["pricing"] == { "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, - } + }, f"a catalog-declared zero is free, not unknown, got {response['pricing']}" def test_create_model_info_response_configured_pricing_overrides_cost_map(): @@ -1242,7 +1242,7 @@ def test_create_model_info_response_configured_pricing_overrides_cost_map(): assert response["pricing"] == { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, - } + }, f"configured pricing must outrank the cost map, got {response['pricing']}" def test_create_model_info_response_reports_highest_price_in_a_mixed_group(): @@ -1331,7 +1331,7 @@ def test_create_model_info_response_does_not_let_a_custom_rate_hide_a_dearer_cat assert response["pricing"] == { "input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05, - } + }, f"the uncustomised deployment is dearer and must set the price, got {response['pricing']}" def test_create_model_info_response_prefers_a_cheaper_override_over_its_own_catalog_price(): @@ -1363,7 +1363,7 @@ def test_create_model_info_response_prefers_a_cheaper_override_over_its_own_cata assert response["pricing"] == { "input_cost_per_token": 3e-08, "output_cost_per_token": 4e-08, - } + }, f"the override is what the request bills, even when cheaper, got {response['pricing']}" def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): @@ -1394,7 +1394,7 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): assert response["pricing"] == { "input_cost_per_token": 9e-06, "output_cost_per_token": 9e-05, - } + }, f"expected the wildcard override, got {response['pricing']}" def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern(): @@ -1428,7 +1428,7 @@ def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pa assert response["pricing"] == { "input_cost_per_token": 4e-05, "output_cost_per_token": 9e-05, - } + }, f"expected the dearest matched deployment per field, got {response['pricing']}" def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog(): @@ -1457,7 +1457,7 @@ def test_create_model_info_response_wildcard_sibling_without_an_override_still_b assert response["pricing"] == { "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, - } + }, f"the sibling without an override bills catalog and is dearest, got {response['pricing']}" def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row(): @@ -1480,7 +1480,7 @@ def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildca assert response["pricing"] == { "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, - } + }, f"an unpriced wildcard leaves the catalog answer, got {response['pricing']}" def test_create_model_info_response_reports_unknown_price_as_null(): @@ -1496,7 +1496,7 @@ def test_create_model_info_response_reports_unknown_price_as_null(): assert response["pricing"] == { "input_cost_per_token": None, "output_cost_per_token": None, - } + }, f"an unmapped model must read as unknown, got {response['pricing']}" def test_create_model_info_response_ignores_malformed_price(): @@ -1519,7 +1519,7 @@ def test_create_model_info_response_ignores_malformed_price(): assert response["pricing"] == { "input_cost_per_token": None, "output_cost_per_token": None, - } + }, f"malformed and negative values must be dropped, got {response['pricing']}" def test_create_model_info_response_accepts_string_price(): @@ -1538,7 +1538,7 @@ def test_create_model_info_response_accepts_string_price(): assert response["pricing"] == { "input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05, - } + }, f"a string price must reach the listing as a number, got {response['pricing']}" def test_create_model_info_response_pricing_does_not_call_router_group_info(): @@ -1557,7 +1557,7 @@ def test_create_model_info_response_pricing_does_not_call_router_group_info(): ) router.get_model_group_info.assert_not_called() - assert response["pricing"]["input_cost_per_token"] == 2.5e-06 + assert response["pricing"]["input_cost_per_token"] == 2.5e-06, f"expected the cost-map price, got {response['pricing']}" def test_get_model_listing_info_carries_custom_deployment_pricing(): @@ -1595,7 +1595,7 @@ def test_get_model_listing_info_carries_custom_deployment_pricing(): assert response["pricing"] == { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, - } + }, f"expected the deployment's configured pricing, got {response['pricing']}" def test_create_model_info_response_resolves_alias_once_per_listing(): diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b2f81c31d56..0e75996bd32 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9588,7 +9588,7 @@ def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params( assert price == DeploymentListingPrice( cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None - ) + ), f"base_model keys the catalog and litellm_params wins the price, got {price}" def test_get_model_listing_info_keeps_each_deployment_price_separate(): @@ -9620,7 +9620,7 @@ def test_get_model_listing_info_keeps_each_deployment_price_separate(): cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=None ), DeploymentListingPrice(cost_map_key="openai/gpt-5.5", input_cost_per_token=None, output_cost_per_token=None), - ) + ), f"each deployment needs its own record and key, got {info.deployment_prices}" def test_get_wildcard_listing_prices_reports_every_matched_deployment(): @@ -9654,7 +9654,7 @@ def test_get_wildcard_listing_prices_reports_every_matched_deployment(): assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == { (9e-06, 9e-05), (4e-05, 8e-05), - } + }, f"every matched deployment must be returned, got {prices}" def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced(): From 2b7a8e77ae65538e089ace13622a4b2755004cf3 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Mon, 21 Sep 2026 14:57:24 +0530 Subject: [PATCH 6/9] test(pricing): assert the whole default listing entry Review flagged the default-response test for pinning a field's absence. Asserting the entry whole is both what tests/AGENTS.md asks for and a stronger check: it fails for any unexpected key the listing starts emitting, not just for `pricing`. Verified by mutation: adding a stray key to the response fails this test. --- tests/test_litellm/proxy/test_proxy_utils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 9d3421cadc1..97763ccd160 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -987,6 +987,7 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: from typing import cast import litellm +from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME from litellm.proxy.utils import create_model_info_response from litellm.types.router import DeploymentListingPrice, DeploymentModelListingInfo from litellm.types.utils import ModelInfo @@ -1130,7 +1131,11 @@ def test_create_model_info_response_reports_widest_window_in_a_mixed_group(): def test_create_model_info_response_omits_pricing_unless_asked(): - """The default listing shape is unchanged, so existing clients see no new field.""" + """A caller who does not ask for pricing gets the same entry as before the flag. + + Asserted whole rather than as "pricing is absent", so any other key the listing starts + emitting by default fails here too. + """ response = create_model_info_response( model_id="gpt-4o", provider="openai", @@ -1140,7 +1145,12 @@ def test_create_model_info_response_omits_pricing_unless_asked(): ), ) - assert "pricing" not in response + assert response == { + "id": "gpt-4o", + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": "openai", + }, f"the default listing entry changed shape: {response}" def test_create_model_info_response_includes_cost_map_pricing(): From 914da409e8310b5bf88e9fd14f607b0a311f6d91 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Tue, 29 Sep 2026 13:07:37 +0530 Subject: [PATCH 7/9] test(core_helpers): cover every branch of coerce_token_price Codecov reported one uncovered line on the patch. Both early-return guards in coerce_token_price were unexercised: every test reached it indirectly through create_model_info_response and none passed a bool or a non-numeric, non-string value. Neither it nor its sibling coerce_token_limit had a direct test anywhere. Parametrized over every branch, including the two that were missed. The bool case earns its place: True is an int in Python, so without that guard a config value of true would price a token at 1.0. Zero is asserted as a price rather than an absence, which is the distinction the feature rests on. --- .../litellm_core_utils/test_core_helpers.py | 94 +++++++++++++------ 1 file changed, 66 insertions(+), 28 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 2a6dd347d5f..7c85e632856 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.core_helpers import ( RESPONSE_COST_HEADER, bind_budget_reservation_to_callbacks, budget_reservation_from_metadata, + coerce_token_price, drop_params_env_flag, drop_params_flag, get_or_create_metadata_bucket, @@ -183,9 +184,7 @@ class TestMapFinishReasonAnthropic: ("content_filtered", "content_filter"), ], ) - def test_anthropic_finish_reasons( - self, provider_reason: str, expected: str - ) -> None: + def test_anthropic_finish_reasons(self, provider_reason: str, expected: str) -> None: assert map_finish_reason(provider_reason) == expected def test_refusal(self): @@ -256,9 +255,7 @@ class TestMapFinishReasonZhipu: class TestMapFinishReasonOpenAIPassthrough: - @pytest.mark.parametrize( - "reason", ["stop", "length", "tool_calls", "function_call", "content_filter"] - ) + @pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "function_call", "content_filter"]) def test_openai_values_pass_through(self, reason): assert map_finish_reason(reason) == reason @@ -269,9 +266,7 @@ class TestMapFinishReasonGenericError: assert map_finish_reason("error") == "stop" def test_lowercase_error_does_not_warn(self, mocker): - warn = mocker.patch( - "litellm.litellm_core_utils.core_helpers.verbose_logger.warning" - ) + warn = mocker.patch("litellm.litellm_core_utils.core_helpers.verbose_logger.warning") assert map_finish_reason("error") == "stop" warn.assert_not_called() @@ -289,8 +284,7 @@ class TestFinishReasonMapOutputsAreValid: """Every value in _FINISH_REASON_MAP must be a valid OpenAI finish reason.""" for provider_reason, openai_reason in _FINISH_REASON_MAP.items(): assert openai_reason in VALID_OPENAI_FINISH_REASONS, ( - f"Mapped value '{openai_reason}' (from '{provider_reason}') " - f"is not a valid OpenAI finish reason" + f"Mapped value '{openai_reason}' (from '{provider_reason}') is not a valid OpenAI finish reason" ) @@ -300,28 +294,18 @@ class TestRedactNestedMatchAndRegexKeys: "assessments": [ { "sensitiveInformationPolicy": { - "piiEntities": [ - {"type": "NAME", "match": "secret-name", "action": "BLOCKED"} - ] - }, - "wordPolicy": { - "customWords": [{"match": "badword", "action": "BLOCKED"}] + "piiEntities": [{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}] }, + "wordPolicy": {"customWords": [{"match": "badword", "action": "BLOCKED"}]}, } ], "regex": "should-redact-key-named-regex", } out = redact_nested_match_and_regex_keys(payload) - assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ - "match" - ] == "[REDACTED]" - assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ( - "[REDACTED]" - ) + assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "[REDACTED]" + assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ("[REDACTED]") assert out["regex"] == "[REDACTED]" - assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][ - 0 - ]["match"] == "secret-name" + assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "secret-name" def test_passes_through_none_and_str(self): assert redact_nested_match_and_regex_keys(None) is None @@ -484,13 +468,17 @@ class TestIsExpectedClientError: assert is_expected_client_error(over_budget) is True litellm_limit = RateLimitError( - message="key over rpm", llm_provider="anthropic", model="claude-haiku-4-5", + message="key over rpm", + llm_provider="anthropic", + model="claude-haiku-4-5", category=RateLimitErrorCategory.LITELLM_RATE_LIMIT, ) assert is_expected_client_error(litellm_limit) is True vendor_limit = RateLimitError( - message="rate limited upstream", llm_provider="anthropic", model="claude-haiku-4-5", + message="rate limited upstream", + llm_provider="anthropic", + model="claude-haiku-4-5", category=RateLimitErrorCategory.VENDOR_RATE_LIMIT, ) assert is_expected_client_error(vendor_limit) is False @@ -557,3 +545,53 @@ class TestProviderResponseHeadersInHiddenParams: assert get_provider_response_headers_from_hidden_params(sibling) is None assert "additional_headers" not in sibling._hidden_params + + +class TestCoerceTokenPrice: + """Per-token prices reach the listing from the same uncoerced sources as token + limits, since a deployment's model_info is registered into litellm.model_cost + verbatim. A config value written as a string has to survive, while anything that + cannot be a price has to read as absent rather than as free.""" + + @pytest.mark.parametrize( + "value, expected", + [ + (0.000003, 0.000003), + (3, 3.0), + ("0.000003", 0.000003), + ("3", 3.0), + (0, 0.0), + ("0", 0.0), + ], + ) + def test_a_usable_price_survives_as_a_float(self, value, expected): + assert coerce_token_price(value) == expected, f"{value!r} should coerce to {expected}" + + def test_zero_is_a_price_not_an_absence(self): + """A deployment priced at zero is deliberately free, which is not the same as + a model nobody has priced.""" + assert coerce_token_price(0) == 0.0, "an explicit zero must not read as absent" + assert coerce_token_price(0) is not None, "an explicit zero must not read as absent" + + @pytest.mark.parametrize("value", [True, False]) + def test_a_bool_is_rejected(self, value): + """True is an int in Python, so without this guard it would price a token at 1.""" + assert coerce_token_price(value) is None, f"{value!r} is never a meaningful price" + + @pytest.mark.parametrize("value", [None, [], {}, (), object(), b"3"]) + def test_a_value_that_is_not_a_number_or_string_is_rejected(self, value): + assert coerce_token_price(value) is None, f"{value!r} cannot be a price" + + @pytest.mark.parametrize("value", ["", "abc", "1.2.3", " "]) + def test_a_string_that_is_not_a_number_is_rejected(self, value): + assert coerce_token_price(value) is None, f"{value!r} must not fail the listing" + + @pytest.mark.parametrize("value", [-1, -0.5, "-0.000003"]) + def test_a_negative_price_is_rejected(self, value): + """A caller pricing a request would read a negative rate as a discount.""" + assert coerce_token_price(value) is None, f"{value!r} must not be reported as a price" + + @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), "nan", "inf"]) + def test_a_non_finite_price_is_rejected(self, value): + """NaN and infinity serialize to invalid JSON.""" + assert coerce_token_price(value) is None, f"{value!r} must not reach the response body" From 7e031f007df55fd0239de50ce4bf215c40408399 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Tue, 29 Sep 2026 15:53:10 +0530 Subject: [PATCH 8/9] test(router): cover the wildcard pattern deployment scan router_code_coverage.py requires every function in router.py to be called by name from a router-named test file, and _pattern_deployments had no caller. It only exists because splitting a stacked comprehension to satisfy LIT014 turned an inline loop into a function, so one repo rule created work for another. The test asserts the helper's contract rather than its shape: the wildcard price scan reads through it to decide whether any pattern configures a price at all, so missing a deployment would skip pattern matching and quote the catalog for one that overrides it. Three deployments across two patterns, all visited. --- tests/unit/test_router/test_router.py | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 968fa7cf021..0006425b7e2 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -10109,6 +10109,51 @@ def test_get_model_listing_info_keeps_each_deployment_price_separate(): ), f"each deployment needs its own record and key, got {info.deployment_prices}" +def test_pattern_deployments_yields_every_deployment_behind_every_pattern(): + """The wildcard price scan reads through this to decide whether any pattern + configures a price at all, so it has to see all of them: missing one would skip + pattern matching and quote the catalog for a deployment that overrides it.""" + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "k", + "input_cost_per_token": 9e-06, + }, + }, + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "k", + "input_cost_per_token": 4e-05, + }, + }, + { + "model_name": "anthropic/*", + "litellm_params": { + "model": "anthropic/*", + "api_key": "k", + "input_cost_per_token": 2e-05, + }, + }, + ] + ) + + seen = [ + deployment.get("litellm_params", {}).get("input_cost_per_token") + for deployment in router._pattern_deployments() + ] + + assert sorted(price for price in seen if price) == [ + 9e-06, + 2e-05, + 4e-05, + ], f"every deployment behind every pattern must be visited, saw {seen}" + + def test_get_wildcard_listing_prices_reports_every_matched_deployment(): """A pattern can front several deployments and routing can pick any of them, so the listing needs all their prices, not the first one's: the caller reports the dearest.""" From 64bbc3df04eb084103f8c66dc7b343dcf6d7c158 Mon Sep 17 00:00:00 2001 From: ajitsharmas2007 Date: Tue, 29 Sep 2026 16:05:41 +0530 Subject: [PATCH 9/9] style(tests): undo an accidental reformat of test_core_helpers.py Adding the coerce_token_price tests ran ruff format over the whole file, which collapsed 28 lines of unrelated multi-line expressions the change never touched. Nothing required it: make check formats litellm/ files, while the test tree is only linted with ruff check --config ruff-tests.toml. The file is restored to upstream and the test class re-added on its own, so the diff is 51 insertions and no deletions. A reviewer can now see which lines are the change. --- .../litellm_core_utils/test_core_helpers.py | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 7c85e632856..6d393558e9d 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -184,7 +184,9 @@ class TestMapFinishReasonAnthropic: ("content_filtered", "content_filter"), ], ) - def test_anthropic_finish_reasons(self, provider_reason: str, expected: str) -> None: + def test_anthropic_finish_reasons( + self, provider_reason: str, expected: str + ) -> None: assert map_finish_reason(provider_reason) == expected def test_refusal(self): @@ -255,7 +257,9 @@ class TestMapFinishReasonZhipu: class TestMapFinishReasonOpenAIPassthrough: - @pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "function_call", "content_filter"]) + @pytest.mark.parametrize( + "reason", ["stop", "length", "tool_calls", "function_call", "content_filter"] + ) def test_openai_values_pass_through(self, reason): assert map_finish_reason(reason) == reason @@ -266,7 +270,9 @@ class TestMapFinishReasonGenericError: assert map_finish_reason("error") == "stop" def test_lowercase_error_does_not_warn(self, mocker): - warn = mocker.patch("litellm.litellm_core_utils.core_helpers.verbose_logger.warning") + warn = mocker.patch( + "litellm.litellm_core_utils.core_helpers.verbose_logger.warning" + ) assert map_finish_reason("error") == "stop" warn.assert_not_called() @@ -284,7 +290,8 @@ class TestFinishReasonMapOutputsAreValid: """Every value in _FINISH_REASON_MAP must be a valid OpenAI finish reason.""" for provider_reason, openai_reason in _FINISH_REASON_MAP.items(): assert openai_reason in VALID_OPENAI_FINISH_REASONS, ( - f"Mapped value '{openai_reason}' (from '{provider_reason}') is not a valid OpenAI finish reason" + f"Mapped value '{openai_reason}' (from '{provider_reason}') " + f"is not a valid OpenAI finish reason" ) @@ -294,18 +301,28 @@ class TestRedactNestedMatchAndRegexKeys: "assessments": [ { "sensitiveInformationPolicy": { - "piiEntities": [{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}] + "piiEntities": [ + {"type": "NAME", "match": "secret-name", "action": "BLOCKED"} + ] + }, + "wordPolicy": { + "customWords": [{"match": "badword", "action": "BLOCKED"}] }, - "wordPolicy": {"customWords": [{"match": "badword", "action": "BLOCKED"}]}, } ], "regex": "should-redact-key-named-regex", } out = redact_nested_match_and_regex_keys(payload) - assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "[REDACTED]" - assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ("[REDACTED]") + assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ + "match" + ] == "[REDACTED]" + assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ( + "[REDACTED]" + ) assert out["regex"] == "[REDACTED]" - assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "secret-name" + assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][ + 0 + ]["match"] == "secret-name" def test_passes_through_none_and_str(self): assert redact_nested_match_and_regex_keys(None) is None @@ -468,17 +485,13 @@ class TestIsExpectedClientError: assert is_expected_client_error(over_budget) is True litellm_limit = RateLimitError( - message="key over rpm", - llm_provider="anthropic", - model="claude-haiku-4-5", + message="key over rpm", llm_provider="anthropic", model="claude-haiku-4-5", category=RateLimitErrorCategory.LITELLM_RATE_LIMIT, ) assert is_expected_client_error(litellm_limit) is True vendor_limit = RateLimitError( - message="rate limited upstream", - llm_provider="anthropic", - model="claude-haiku-4-5", + message="rate limited upstream", llm_provider="anthropic", model="claude-haiku-4-5", category=RateLimitErrorCategory.VENDOR_RATE_LIMIT, ) assert is_expected_client_error(vendor_limit) is False