mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
Merge 0e0a309dbb into 82a5d9842a
This commit is contained in:
commit
85015e7f1f
10 changed files with 1017 additions and 3 deletions
|
|
@ -2,6 +2,7 @@
|
|||
## Helper utilities
|
||||
import copy
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Collection, Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
|
|
@ -195,6 +196,31 @@ def coerce_token_limit(value: object) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def coerce_token_price(value: object) -> float | None:
|
||||
"""
|
||||
Coerce a per-token price to a float, treating a malformed value as absent.
|
||||
|
||||
Prices reach the /v1/models listing from the same uncoerced sources as token limits
|
||||
(a deployment's model_info is registered into litellm.model_cost verbatim), so a
|
||||
config value like "0.000003" has to survive while "" does not fail the listing.
|
||||
|
||||
Negative and non-finite values are rejected: a caller pricing a request would read a
|
||||
negative rate as a discount, and NaN/inf serialize to invalid JSON. Bools are rejected
|
||||
because True/False is never a meaningful price.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if not isinstance(value, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
price: Final = float(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
if not math.isfinite(price) or price < 0:
|
||||
return None
|
||||
return price
|
||||
|
||||
|
||||
_FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
|
||||
# Anthropic
|
||||
"stop_sequence": "stop",
|
||||
|
|
|
|||
|
|
@ -11853,6 +11853,7 @@ async def model_list(
|
|||
scope: str | None = None,
|
||||
healthy_only: bool | None = False,
|
||||
client_version: str | None = None,
|
||||
include_pricing: bool | None = False,
|
||||
):
|
||||
"""
|
||||
Use `/model/info` - to get detailed model information, example - pricing, mode, etc.
|
||||
|
|
@ -11866,6 +11867,13 @@ async def model_list(
|
|||
each model's `model_info.service_tiers` as its service tiers; absent,
|
||||
the OpenAI shape below is returned
|
||||
- include_metadata: Include additional metadata in the response with fallback information
|
||||
- include_pricing: Include a `pricing` object per model, carrying
|
||||
`input_cost_per_token` and `output_cost_per_token` in USD per token.
|
||||
Custom pricing configured on a deployment takes precedence over the
|
||||
built-in cost map. A price the proxy does not know is reported as
|
||||
`null`, never 0, so an unmapped model is not read as free. When a
|
||||
model group's deployments disagree on price, the highest is
|
||||
reported, matching `/model_group/info`.
|
||||
- fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
|
||||
Defaults to "general" when include_metadata=true
|
||||
- scope: Optional scope parameter. Currently only accepts "expand".
|
||||
|
|
@ -12005,6 +12013,7 @@ async def model_list(
|
|||
include_metadata=include_metadata or False,
|
||||
fallback_type=fallback_type,
|
||||
llm_router=llm_router,
|
||||
include_pricing=include_pricing or False,
|
||||
)
|
||||
model_info["id"] = response_id
|
||||
model_data.append(model_info)
|
||||
|
|
@ -12073,6 +12082,7 @@ async def model_list(
|
|||
include_metadata=include_metadata or False,
|
||||
fallback_type=fallback_type,
|
||||
llm_router=llm_router,
|
||||
include_pricing=include_pricing or False,
|
||||
)
|
||||
model_info["id"] = response_id
|
||||
model_data.append(model_info)
|
||||
|
|
|
|||
|
|
@ -81,7 +81,8 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse, ModelPricing
|
||||
from litellm.types.router import DeploymentListingPrice
|
||||
from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage
|
||||
|
||||
try:
|
||||
|
|
@ -143,6 +144,7 @@ from litellm.integrations.SlackAlerting.utils import add_langfuse_trace_id_to_al
|
|||
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
coerce_token_price,
|
||||
get_or_create_metadata_bucket,
|
||||
independent_snapshot,
|
||||
is_expected_client_error,
|
||||
|
|
@ -286,6 +288,7 @@ if TYPE_CHECKING:
|
|||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
|
||||
from litellm.types.router import DeploymentModelListingInfo
|
||||
|
||||
Span = _Span | object
|
||||
else:
|
||||
|
|
@ -8952,6 +8955,138 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field:
|
|||
return max(limits) if limits else None
|
||||
|
||||
|
||||
def _declared_price(info: ModelInfo, field: str, cost_map: Mapping[str, object]) -> float | None:
|
||||
"""The price ``field`` carries, but only when the cost map actually declares it.
|
||||
|
||||
A resolved ``ModelInfo`` cannot be trusted on its own here. Once the router registers
|
||||
a sparse cost-map entry for a deployment the catalog has never priced, the lookup
|
||||
stops raising and defaults the missing costs to 0, so an unpriced model would be
|
||||
served as free. The raw entry is the authority on whether a price exists at all,
|
||||
which is the same test ``_is_cost_explicitly_configured`` makes for budget checks.
|
||||
|
||||
A model the catalog deliberately prices at 0.0 still reports 0.0; one it never priced
|
||||
reports None.
|
||||
"""
|
||||
key: Final = info.get("key")
|
||||
raw: Final = cost_map.get(key) if isinstance(key, str) else None
|
||||
if not isinstance(raw, Mapping) or field not in raw:
|
||||
return None
|
||||
return coerce_token_price(info.get(field, raw[field]))
|
||||
|
||||
|
||||
def _first_price(candidates: tuple[ModelInfo, ...], field: str, cost_map: Mapping[str, object]) -> float | None:
|
||||
"""The price ``field`` takes from the best cost-map entry describing one deployment.
|
||||
|
||||
``candidates`` is best-source-first (the exact entry for the deployment's underlying
|
||||
model before a generalized family fallback), so these are not competing prices to
|
||||
compare: it is one price described with varying reliability, and the first usable
|
||||
answer wins.
|
||||
"""
|
||||
return next(
|
||||
(price for price in (_declared_price(info, field, cost_map) for info in candidates) if price is not None),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _listing_deployment_prices(
|
||||
lookup_model: str,
|
||||
listing_info: "DeploymentModelListingInfo | None",
|
||||
llm_router: Optional["Router"],
|
||||
) -> tuple["DeploymentListingPrice", ...]:
|
||||
"""The per-deployment prices behind a listed name, one entry per deployment.
|
||||
|
||||
A name the router indexes carries its deployments' own records. A wildcard-expanded
|
||||
name has none, so the overrides live on the patterns it expands from and are asked for
|
||||
there, one record per matched deployment because a request can route to any of them.
|
||||
Those records are keyed to None so a deployment that configures nothing, or configures
|
||||
only one half, still falls back to the catalog entry the listing already resolved.
|
||||
"""
|
||||
if listing_info is not None:
|
||||
return listing_info.deployment_prices
|
||||
|
||||
wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else ()
|
||||
if not wildcard_prices:
|
||||
return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),)
|
||||
return tuple(
|
||||
DeploymentListingPrice(
|
||||
cost_map_key=None,
|
||||
input_cost_per_token=price.input_cost_per_token,
|
||||
output_cost_per_token=price.output_cost_per_token,
|
||||
)
|
||||
for price in wildcard_prices
|
||||
)
|
||||
|
||||
|
||||
def _deployment_effective_price(
|
||||
configured: float | None,
|
||||
cost_map_key: str | None,
|
||||
field: str,
|
||||
candidates_by_key: Mapping[str | None, tuple[ModelInfo, ...]],
|
||||
cost_map: Mapping[str, object],
|
||||
) -> float | None:
|
||||
"""What one deployment charges for ``field``: its own override, else its catalog entry."""
|
||||
if configured is not None:
|
||||
return configured
|
||||
return _first_price(candidates_by_key.get(cost_map_key, ()), field, cost_map)
|
||||
|
||||
|
||||
def _highest_price(prices: tuple[float | None, ...]) -> float | None:
|
||||
known: Final = tuple(price for price in prices if price is not None)
|
||||
return max(known) if known else None
|
||||
|
||||
|
||||
def _listing_pricing(
|
||||
candidate_sets: tuple[tuple[ModelInfo, ...], ...],
|
||||
deployment_models: tuple[str | None, ...],
|
||||
deployment_prices: tuple["DeploymentListingPrice", ...],
|
||||
cost_map: Mapping[str, object],
|
||||
) -> ModelPricing:
|
||||
"""Effective per-token prices for a listed model, in USD.
|
||||
|
||||
Each deployment resolves its own price first, its override when it has one and its
|
||||
catalog entry otherwise, and the group reports the highest of those. Comparing
|
||||
overrides against each other and only then falling back to the catalog would hide a
|
||||
dearer deployment that happens to configure no override, quoting a caller less than
|
||||
the request that lands there actually costs.
|
||||
|
||||
The highest is the deliberate pick across a group, matching what ``/model_group/info``
|
||||
reports, because the listing cannot know which deployment a later request will reach.
|
||||
|
||||
A price the proxy does not know is reported as None rather than 0, so a caller doing
|
||||
cost accounting cannot read an unmapped model as free.
|
||||
"""
|
||||
candidates_by_key: Final[Mapping[str | None, tuple[ModelInfo, ...]]] = MappingProxyType(
|
||||
{key: candidates for key, candidates in zip(deployment_models, candidate_sets)}
|
||||
)
|
||||
pricing: Final[ModelPricing] = {
|
||||
"input_cost_per_token": _highest_price(
|
||||
tuple(
|
||||
_deployment_effective_price(
|
||||
deployment.input_cost_per_token,
|
||||
deployment.cost_map_key,
|
||||
"input_cost_per_token",
|
||||
candidates_by_key,
|
||||
cost_map,
|
||||
)
|
||||
for deployment in deployment_prices
|
||||
)
|
||||
),
|
||||
"output_cost_per_token": _highest_price(
|
||||
tuple(
|
||||
_deployment_effective_price(
|
||||
deployment.output_cost_per_token,
|
||||
deployment.cost_map_key,
|
||||
"output_cost_per_token",
|
||||
candidates_by_key,
|
||||
cost_map,
|
||||
)
|
||||
for deployment in deployment_prices
|
||||
)
|
||||
),
|
||||
}
|
||||
return pricing
|
||||
|
||||
|
||||
def create_model_info_response(
|
||||
model_id: str,
|
||||
provider: str,
|
||||
|
|
@ -8959,6 +9094,8 @@ def create_model_info_response(
|
|||
fallback_type: str | None = None,
|
||||
llm_router: Optional["Router"] = None,
|
||||
get_model_info: Callable[[str], ModelInfo] = litellm.get_model_info,
|
||||
include_pricing: bool = False,
|
||||
cost_map: Mapping[str, object] | None = None,
|
||||
) -> ModelInfoResponse:
|
||||
"""
|
||||
Create a standardized OpenAI-compatible model object.
|
||||
|
|
@ -8966,6 +9103,11 @@ def create_model_info_response(
|
|||
When include_metadata is true, attaches the model's configured fallbacks
|
||||
(resolved via the router under fallback_type, defaulting to "general").
|
||||
Raises HTTPException(400) for an unknown fallback_type.
|
||||
|
||||
When include_pricing is true, attaches the model's effective per-token prices in USD,
|
||||
resolved from the same deployment-then-cost-map sources as the token limits above.
|
||||
cost_map defaults to litellm.model_cost, resolved per call because a price reload
|
||||
rebuilds it.
|
||||
"""
|
||||
from litellm.proxy.auth.model_checks import get_all_fallbacks
|
||||
|
||||
|
|
@ -9032,6 +9174,14 @@ def create_model_info_response(
|
|||
if max_output_tokens is not None:
|
||||
base["max_output_tokens"] = max_output_tokens
|
||||
|
||||
if include_pricing:
|
||||
base["pricing"] = _listing_pricing(
|
||||
candidate_sets,
|
||||
deployment_models,
|
||||
_listing_deployment_prices(lookup_model, listing_info, llm_router),
|
||||
cost_map if cost_map is not None else litellm.model_cost,
|
||||
)
|
||||
|
||||
if not include_metadata:
|
||||
return base
|
||||
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ from litellm.integrations.otel.runtime import phase_attributes, phase_event, pha
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
coerce_token_price,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
get_or_create_metadata_bucket,
|
||||
|
|
@ -302,6 +303,7 @@ from litellm.types.router import (
|
|||
CredentialLiteLLMParams,
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
DeploymentListingPrice,
|
||||
DeploymentModelListingInfo,
|
||||
DeploymentTypedDict,
|
||||
DiscoveredDeploymentModelInfo,
|
||||
|
|
@ -10796,6 +10798,10 @@ class Router:
|
|||
cost_map_keys=cost_map_keys,
|
||||
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),
|
||||
max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
|
||||
deployment_prices=tuple(
|
||||
self._deployment_listing_price(model_info, litellm_params)
|
||||
for model_info, litellm_params in zip(model_infos, params)
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -10808,6 +10814,76 @@ class Router:
|
|||
)
|
||||
return max(limits) if limits else None
|
||||
|
||||
@staticmethod
|
||||
def _configured_price(
|
||||
model_info: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, Any],
|
||||
field: str,
|
||||
) -> float | None:
|
||||
"""One deployment's custom price for ``field``, or None when it configures none.
|
||||
|
||||
Both sources are read because custom pricing is accepted in ``model_info`` and in
|
||||
``litellm_params``; the router itself copies the latter onto the former when it
|
||||
builds a deployment's cost-map entry. ``litellm_params`` wins, matching that copy.
|
||||
"""
|
||||
from_params: Final = coerce_token_price(litellm_params.get(field))
|
||||
return from_params if from_params is not None else coerce_token_price(model_info.get(field))
|
||||
|
||||
@staticmethod
|
||||
def _deployment_listing_price(
|
||||
model_info: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, Any],
|
||||
) -> DeploymentListingPrice:
|
||||
"""What one deployment contributes to its listed name's price."""
|
||||
cost_map_key: Final = (
|
||||
model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
|
||||
)
|
||||
return DeploymentListingPrice(
|
||||
cost_map_key=cost_map_key if isinstance(cost_map_key, str) and cost_map_key else None,
|
||||
input_cost_per_token=Router._configured_price(model_info, litellm_params, "input_cost_per_token"),
|
||||
output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"),
|
||||
)
|
||||
|
||||
def _pattern_deployments(self) -> Iterator[Mapping[str, object]]:
|
||||
"""Every deployment registered behind a wildcard pattern, across all patterns."""
|
||||
for deployments in self.pattern_router.patterns.values():
|
||||
yield from deployments
|
||||
|
||||
def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]:
|
||||
"""One record per wildcard deployment that a name expands from, in match order.
|
||||
|
||||
Wildcard-expanded rows are absent from the model-name index, so their deployments'
|
||||
overrides are invisible to ``get_model_listing_info`` and the listing would quote
|
||||
the catalog while the request is billed at an override.
|
||||
|
||||
Every matched deployment is returned rather than the first, because a request can
|
||||
route to any of them: the caller resolves each one and reports the dearest, the same
|
||||
way it treats a group the index does know.
|
||||
|
||||
Pattern matching is the expensive path this listing exists to avoid, so it only
|
||||
runs when a wildcard deployment actually configures a price: with none, the catalog
|
||||
is already the right answer and nothing here can change it.
|
||||
"""
|
||||
if not any(
|
||||
price.input_cost_per_token is not None or price.output_cost_per_token is not None
|
||||
for price in (
|
||||
Router._deployment_listing_price(
|
||||
deployment.get("model_info") or MappingProxyType({}),
|
||||
deployment.get("litellm_params") or MappingProxyType({}),
|
||||
)
|
||||
for deployment in self._pattern_deployments()
|
||||
)
|
||||
):
|
||||
return ()
|
||||
|
||||
return tuple(
|
||||
Router._deployment_listing_price(
|
||||
deployment.get("model_info") or MappingProxyType({}),
|
||||
deployment.get("litellm_params") or MappingProxyType({}),
|
||||
)
|
||||
for deployment in (self.pattern_router.route(model_name) or ())
|
||||
)
|
||||
|
||||
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
|
||||
"""
|
||||
Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete
|
||||
|
|
|
|||
|
|
@ -2,13 +2,24 @@
|
|||
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import NotRequired, TypedDict
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
|
||||
class ModelInfoMetadata(TypedDict):
|
||||
fallbacks: list[str]
|
||||
|
||||
|
||||
class ModelPricing(TypedDict):
|
||||
"""Effective token prices for a listed model, in USD per token.
|
||||
|
||||
``None`` is not the same as free: an unmapped model reports null so a caller doing
|
||||
cost accounting cannot read a missing price as a zero-cost model.
|
||||
"""
|
||||
|
||||
input_cost_per_token: ReadOnly[float | None]
|
||||
output_cost_per_token: ReadOnly[float | None]
|
||||
|
||||
|
||||
class ModelInfoResponse(TypedDict):
|
||||
"""OpenAI-compatible model object. `mode`, `max_input_tokens`, and
|
||||
`max_output_tokens` are attached when the cost map or deployment config
|
||||
|
|
@ -23,3 +34,4 @@ class ModelInfoResponse(TypedDict):
|
|||
max_input_tokens: NotRequired[int]
|
||||
max_output_tokens: NotRequired[int]
|
||||
metadata: NotRequired[ModelInfoMetadata]
|
||||
pricing: NotRequired[ModelPricing]
|
||||
|
|
|
|||
|
|
@ -789,6 +789,22 @@ class DiscoveredDeploymentModelInfo:
|
|||
limits: Mapping[str, int]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentListingPrice:
|
||||
"""What one deployment behind a listed name contributes to its price.
|
||||
|
||||
``cost_map_key`` names the catalog entry that deployment resolves to, so a caller can
|
||||
fall back to the catalog for a deployment that configures no price of its own. The
|
||||
prices are whatever that single deployment configures, never a group aggregate: a
|
||||
group's price is only correct once each member resolves its own effective price
|
||||
first, since a deployment without an override can still be the dearest one.
|
||||
"""
|
||||
|
||||
cost_map_key: str | None
|
||||
input_cost_per_token: float | None
|
||||
output_cost_per_token: float | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentModelListingInfo:
|
||||
"""What the deployments behind a model name contribute to its OpenAI-compatible listing entry.
|
||||
|
|
@ -800,11 +816,18 @@ class DeploymentModelListingInfo:
|
|||
order, so the ordinary group -- several interchangeable deployments of one model --
|
||||
carries exactly one. The token limits are the widest explicitly set in any
|
||||
deployment's ``model_info``, which outrank anything the cost map says.
|
||||
|
||||
``deployment_prices`` carries one entry per deployment, in config order, rather than a
|
||||
group aggregate: the cost map cannot supply custom pricing on its own, because a
|
||||
deployment's override is registered only under its own model id and is deliberately
|
||||
stripped from the shared backend key so one deployment's override never becomes
|
||||
another's price.
|
||||
"""
|
||||
|
||||
cost_map_keys: tuple[str, ...]
|
||||
max_input_tokens: int | None
|
||||
max_output_tokens: int | None
|
||||
deployment_prices: tuple[DeploymentListingPrice, ...] = ()
|
||||
|
||||
|
||||
class RouterErrors(enum.Enum):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
RESPONSE_COST_HEADER,
|
||||
bind_budget_reservation_to_callbacks,
|
||||
budget_reservation_from_metadata,
|
||||
coerce_token_price,
|
||||
drop_params_env_flag,
|
||||
drop_params_flag,
|
||||
get_parent_otel_span_from_kwargs,
|
||||
|
|
@ -586,3 +587,53 @@ class TestProviderResponseHeadersInHiddenParams:
|
|||
|
||||
assert get_provider_response_headers_from_hidden_params(sibling) is None
|
||||
assert "additional_headers" not in sibling._hidden_params
|
||||
|
||||
|
||||
class TestCoerceTokenPrice:
|
||||
"""Per-token prices reach the listing from the same uncoerced sources as token
|
||||
limits, since a deployment's model_info is registered into litellm.model_cost
|
||||
verbatim. A config value written as a string has to survive, while anything that
|
||||
cannot be a price has to read as absent rather than as free."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value, expected",
|
||||
[
|
||||
(0.000003, 0.000003),
|
||||
(3, 3.0),
|
||||
("0.000003", 0.000003),
|
||||
("3", 3.0),
|
||||
(0, 0.0),
|
||||
("0", 0.0),
|
||||
],
|
||||
)
|
||||
def test_a_usable_price_survives_as_a_float(self, value, expected):
|
||||
assert coerce_token_price(value) == expected, f"{value!r} should coerce to {expected}"
|
||||
|
||||
def test_zero_is_a_price_not_an_absence(self):
|
||||
"""A deployment priced at zero is deliberately free, which is not the same as
|
||||
a model nobody has priced."""
|
||||
assert coerce_token_price(0) == 0.0, "an explicit zero must not read as absent"
|
||||
assert coerce_token_price(0) is not None, "an explicit zero must not read as absent"
|
||||
|
||||
@pytest.mark.parametrize("value", [True, False])
|
||||
def test_a_bool_is_rejected(self, value):
|
||||
"""True is an int in Python, so without this guard it would price a token at 1."""
|
||||
assert coerce_token_price(value) is None, f"{value!r} is never a meaningful price"
|
||||
|
||||
@pytest.mark.parametrize("value", [None, [], {}, (), object(), b"3"])
|
||||
def test_a_value_that_is_not_a_number_or_string_is_rejected(self, value):
|
||||
assert coerce_token_price(value) is None, f"{value!r} cannot be a price"
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "abc", "1.2.3", " "])
|
||||
def test_a_string_that_is_not_a_number_is_rejected(self, value):
|
||||
assert coerce_token_price(value) is None, f"{value!r} must not fail the listing"
|
||||
|
||||
@pytest.mark.parametrize("value", [-1, -0.5, "-0.000003"])
|
||||
def test_a_negative_price_is_rejected(self, value):
|
||||
"""A caller pricing a request would read a negative rate as a discount."""
|
||||
assert coerce_token_price(value) is None, f"{value!r} must not be reported as a price"
|
||||
|
||||
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), "nan", "inf"])
|
||||
def test_a_non_finite_price_is_rejected(self, value):
|
||||
"""NaN and infinity serialize to invalid JSON."""
|
||||
assert coerce_token_price(value) is None, f"{value!r} must not reach the response body"
|
||||
|
|
|
|||
|
|
@ -989,8 +989,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
|
|||
from typing import cast
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME
|
||||
from litellm.proxy.utils import create_model_info_response
|
||||
from litellm.types.router import DeploymentModelListingInfo
|
||||
from litellm.types.router import DeploymentListingPrice, DeploymentModelListingInfo
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
|
|
@ -1131,6 +1132,484 @@ def test_create_model_info_response_reports_widest_window_in_a_mixed_group():
|
|||
assert response["max_output_tokens"] == 128000, keys
|
||||
|
||||
|
||||
def test_create_model_info_response_omits_pricing_unless_asked():
|
||||
"""A caller who does not ask for pricing gets the same entry as before the flag.
|
||||
|
||||
Asserted whole rather than as "pricing is absent", so any other key the listing starts
|
||||
emitting by default fails here too.
|
||||
"""
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
input_cost_per_token=2.5e-06, output_cost_per_token=1e-05
|
||||
),
|
||||
)
|
||||
|
||||
assert response == {
|
||||
"id": "gpt-4o",
|
||||
"object": "model",
|
||||
"created": DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
"owned_by": "openai",
|
||||
}, f"the default listing entry changed shape: {response}"
|
||||
|
||||
|
||||
def test_create_model_info_response_includes_cost_map_pricing():
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05
|
||||
),
|
||||
cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
}, f"expected the cost-map prices, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_unpriced_model_as_null_not_zero():
|
||||
"""A model the catalog never priced must report null even though the lookup says 0.
|
||||
|
||||
Once the router registers a sparse cost-map entry for an unmapped deployment, the
|
||||
lookup stops raising and defaults the missing costs to 0. Trusting that value served
|
||||
an unpriced model as free, which is exactly what #35312 asks the listing not to do.
|
||||
"""
|
||||
response = create_model_info_response(
|
||||
model_id="unknown-model",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="openai/some-model-that-does-not-exist",
|
||||
input_cost_per_token=0,
|
||||
output_cost_per_token=0,
|
||||
),
|
||||
cost_map={"openai/some-model-that-does-not-exist": {"id": "deployment-hash"}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": None,
|
||||
"output_cost_per_token": None,
|
||||
}, f"a sparse cost-map entry must read as unknown, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_declared_zero_price_as_zero():
|
||||
"""A model the catalog deliberately prices at 0 is free, and must not report null.
|
||||
|
||||
This is the counterpart to the unpriced case: both resolve to 0, only the raw cost-map
|
||||
entry distinguishes "free" from "nobody ever priced this".
|
||||
"""
|
||||
response = create_model_info_response(
|
||||
model_id="free-model",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="free-model", input_cost_per_token=0.0, output_cost_per_token=0.0
|
||||
),
|
||||
cost_map={"free-model": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
}, f"a catalog-declared zero is free, not unknown, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_configured_pricing_overrides_cost_map():
|
||||
"""Custom pricing is the price the request is billed at, so it outranks the cost map.
|
||||
|
||||
The cost map cannot carry it: the router strips custom pricing from the shared backend
|
||||
key so one deployment's override never becomes another's price.
|
||||
"""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_keys=("gpt-4o",),
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
deployment_prices=(
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="gpt-4o", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="house-gpt",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o", input_cost_per_token=2.5e-06, output_cost_per_token=1e-05
|
||||
),
|
||||
cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
}, f"configured pricing must outrank the cost map, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_highest_price_in_a_mixed_group():
|
||||
"""A group mixing models quotes the dearest member, matching /model_group/info.
|
||||
|
||||
Quoting the cheapest would under-budget every request that lands on the dearest
|
||||
deployment, and the listing cannot know which one a later request will reach.
|
||||
"""
|
||||
prices = {
|
||||
"cheap-model": _fake_model_info(
|
||||
key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07
|
||||
),
|
||||
"dear-model": _fake_model_info(
|
||||
key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05
|
||||
),
|
||||
}
|
||||
cost_map = {
|
||||
"cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07},
|
||||
"dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05},
|
||||
}
|
||||
|
||||
for keys in (("cheap-model", "dear-model"), ("dear-model", "cheap-model")):
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_keys=keys,
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
deployment_prices=tuple(
|
||||
DeploymentListingPrice(cost_map_key=key, input_cost_per_token=None, output_cost_per_token=None)
|
||||
for key in keys
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="house-model",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda model: prices[model],
|
||||
cost_map=cost_map,
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"output_cost_per_token": 3e-05,
|
||||
}, keys
|
||||
|
||||
|
||||
def test_create_model_info_response_does_not_let_a_custom_rate_hide_a_dearer_catalog_one():
|
||||
"""One deployment's override must not stand in for a group that has a dearer member.
|
||||
|
||||
Comparing overrides against each other and only then falling back to the catalog
|
||||
quoted the cheap override here while a request routed to the uncustomised deployment
|
||||
billed 17x that, and disagreed with /model_group/info.
|
||||
"""
|
||||
prices = {
|
||||
"cheap-model": _fake_model_info(key="cheap-model", input_cost_per_token=1e-07, output_cost_per_token=4e-07),
|
||||
"dear-model": _fake_model_info(key="dear-model", input_cost_per_token=5e-06, output_cost_per_token=3e-05),
|
||||
}
|
||||
cost_map = {
|
||||
"cheap-model": {"input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07},
|
||||
"dear-model": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05},
|
||||
}
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_keys=("cheap-model", "dear-model"),
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
deployment_prices=(
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="cheap-model", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
|
||||
),
|
||||
DeploymentListingPrice(cost_map_key="dear-model", input_cost_per_token=None, output_cost_per_token=None),
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="house-model",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda model: prices[model],
|
||||
cost_map=cost_map,
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"output_cost_per_token": 3e-05,
|
||||
}, f"the uncustomised deployment is dearer and must set the price, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_prefers_a_cheaper_override_over_its_own_catalog_price():
|
||||
"""The override is what the request is billed at, so it wins even when it is cheaper.
|
||||
|
||||
The guard against the case above must not turn into "always take the catalog".
|
||||
"""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_keys=("gpt-4o",),
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
deployment_prices=(
|
||||
DeploymentListingPrice(cost_map_key="gpt-4o", input_cost_per_token=3e-08, output_cost_per_token=4e-08),
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="house-model",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
}, f"the override is what the request bills, even when cheaper, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row():
|
||||
"""A wildcard-expanded name has no index entry, so its override lives on the pattern.
|
||||
|
||||
Reading only the index quoted the catalog here while the request billed at the
|
||||
wildcard's own rate.
|
||||
"""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 9e-06,
|
||||
"output_cost_per_token": 9e-05,
|
||||
}, f"expected the wildcard override, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern():
|
||||
"""A pattern can front several deployments, and routing can pick any of them.
|
||||
|
||||
Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing
|
||||
advertised less than the request could cost.
|
||||
"""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
|
||||
),
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 4e-05,
|
||||
"output_cost_per_token": 9e-05,
|
||||
}, f"expected the dearest matched deployment per field, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog():
|
||||
"""A matched deployment configuring nothing is billed at the catalog, so it can be the
|
||||
dearest member even when its sibling carries a very cheap override."""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08
|
||||
),
|
||||
DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
}, f"the sibling without an override bills catalog and is dearest, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row():
|
||||
"""An unpriced wildcard changes nothing, so the catalog answer still stands."""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = ()
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
}, f"an unpriced wildcard leaves the catalog answer, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_unknown_price_as_null():
|
||||
"""An unmapped model must not read as free to a caller doing cost accounting."""
|
||||
response = create_model_info_response(
|
||||
model_id="my-custom-deployment",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
include_pricing=True,
|
||||
get_model_info=_raise_unmapped,
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": None,
|
||||
"output_cost_per_token": None,
|
||||
}, f"an unmapped model must read as unknown, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_ignores_malformed_price():
|
||||
"""A malformed or negative price is dropped rather than served or blowing up the listing.
|
||||
|
||||
model_info is registered into the cost map verbatim, so a config typo reaches here
|
||||
uncoerced; a negative rate would read as a discount and NaN is not valid JSON.
|
||||
"""
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o", input_cost_per_token="", output_cost_per_token=-1e-06
|
||||
),
|
||||
cost_map={"gpt-4o": {"input_cost_per_token": "", "output_cost_per_token": -1e-06}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": None,
|
||||
"output_cost_per_token": None,
|
||||
}, f"malformed and negative values must be dropped, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_accepts_string_price():
|
||||
"""A price configured as a string still reaches the listing as a number."""
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
provider="openai",
|
||||
llm_router=None,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o", input_cost_per_token="0.0000025", output_cost_per_token="0.00001"
|
||||
),
|
||||
cost_map={"gpt-4o": {"input_cost_per_token": "0.0000025", "output_cost_per_token": "0.00001"}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
}, f"a string price must reach the listing as a number, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_pricing_does_not_call_router_group_info():
|
||||
"""Pricing must not pull /v1/models onto the expensive group-info path (#33721)."""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = ()
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(key="gpt-4o", input_cost_per_token=2.5e-06),
|
||||
cost_map={"gpt-4o": {"input_cost_per_token": 2.5e-06}},
|
||||
)
|
||||
|
||||
router.get_model_group_info.assert_not_called()
|
||||
assert response["pricing"]["input_cost_per_token"] == 2.5e-06, f"expected the cost-map price, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_get_model_listing_info_carries_custom_deployment_pricing():
|
||||
"""Custom pricing set in litellm_params reaches the listing through the router index."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "house-gpt",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "test-key",
|
||||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
listing_info = router.get_model_listing_info("house-gpt")
|
||||
|
||||
assert listing_info is not None
|
||||
assert listing_info.deployment_prices == (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="house-gpt",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
}, f"expected the deployment's configured pricing, got {response['pricing']}"
|
||||
|
||||
|
||||
def test_create_model_info_response_resolves_alias_once_per_listing():
|
||||
"""The alias is the same for every deployment in the group, so it is looked up once."""
|
||||
seen: list[str] = []
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ from litellm.types.llms.openai import ChatCompletionRequest
|
|||
from litellm.types.router import (
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
DeploymentListingPrice,
|
||||
DeploymentTypedDict,
|
||||
LiteLLM_Params,
|
||||
ModelGroupInfo,
|
||||
|
|
@ -10512,6 +10513,176 @@ def test_get_model_listing_info_returns_none_for_unknown_name():
|
|||
assert router.get_model_listing_info("not-a-real-model") is None
|
||||
|
||||
|
||||
def test_configured_price_reads_model_info_when_litellm_params_has_none():
|
||||
"""Custom pricing is accepted in both blocks, so model_info alone still counts."""
|
||||
price = litellm.Router._configured_price(
|
||||
{"input_cost_per_token": 3e-07}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token"
|
||||
)
|
||||
|
||||
assert price == 3e-07, f"expected the model_info price, got {price}"
|
||||
|
||||
|
||||
def test_configured_price_is_none_when_the_deployment_configures_none():
|
||||
"""Nothing configured means the cost map answers instead, so this must not report 0."""
|
||||
price = litellm.Router._configured_price(
|
||||
{"mode": "chat"}, {"model": "openai/gpt-4o-mini"}, "input_cost_per_token"
|
||||
)
|
||||
|
||||
assert price is None, f"expected no configured price, got {price}"
|
||||
|
||||
|
||||
def test_configured_price_ignores_a_malformed_configured_value():
|
||||
"""model_info reaches the listing uncoerced, so a config typo must not become a price."""
|
||||
price = litellm.Router._configured_price(
|
||||
{"input_cost_per_token": ""}, {"input_cost_per_token": -1e-06}, "input_cost_per_token"
|
||||
)
|
||||
|
||||
assert price is None, f"expected malformed and negative values to be dropped, got {price}"
|
||||
|
||||
|
||||
def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params():
|
||||
"""The record a deployment contributes: its catalog key, plus its own configured prices.
|
||||
|
||||
base_model names the catalog entry when set, because litellm_params.model can be an
|
||||
opaque backend name, and litellm_params wins over model_info for a price, matching the
|
||||
copy the router makes when it builds the deployment's cost-map entry.
|
||||
"""
|
||||
price = litellm.Router._deployment_listing_price(
|
||||
{"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06},
|
||||
{"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06},
|
||||
)
|
||||
|
||||
assert price == DeploymentListingPrice(
|
||||
cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None
|
||||
), f"base_model keys the catalog and litellm_params wins the price, got {price}"
|
||||
|
||||
|
||||
def test_get_model_listing_info_keeps_each_deployment_price_separate():
|
||||
"""A group's price is only correct once each deployment resolves its own price first.
|
||||
|
||||
Collapsing the group to the highest override here would hide a deployment that
|
||||
configures nothing and is dearer in the catalog, so the listing gets one record per
|
||||
deployment, carrying that deployment's own cost-map key.
|
||||
"""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "test-key",
|
||||
"input_cost_per_token": 3e-07,
|
||||
},
|
||||
},
|
||||
{"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "test-key"}},
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("mixed")
|
||||
|
||||
assert info is not None
|
||||
assert info.deployment_prices == (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=3e-07, output_cost_per_token=None
|
||||
),
|
||||
DeploymentListingPrice(cost_map_key="openai/gpt-5.5", input_cost_per_token=None, output_cost_per_token=None),
|
||||
), f"each deployment needs its own record and key, got {info.deployment_prices}"
|
||||
|
||||
|
||||
def test_pattern_deployments_yields_every_deployment_behind_every_pattern():
|
||||
"""The wildcard price scan reads through this to decide whether any pattern
|
||||
configures a price at all, so it has to see all of them: missing one would skip
|
||||
pattern matching and quote the catalog for a deployment that overrides it."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "k",
|
||||
"input_cost_per_token": 9e-06,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "k",
|
||||
"input_cost_per_token": 4e-05,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "anthropic/*",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/*",
|
||||
"api_key": "k",
|
||||
"input_cost_per_token": 2e-05,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
seen = [
|
||||
deployment.get("litellm_params", {}).get("input_cost_per_token")
|
||||
for deployment in router._pattern_deployments()
|
||||
]
|
||||
|
||||
assert sorted(price for price in seen if price) == [
|
||||
9e-06,
|
||||
2e-05,
|
||||
4e-05,
|
||||
], f"every deployment behind every pattern must be visited, saw {seen}"
|
||||
|
||||
|
||||
def test_get_wildcard_listing_prices_reports_every_matched_deployment():
|
||||
"""A pattern can front several deployments and routing can pick any of them, so the
|
||||
listing needs all their prices, not the first one's: the caller reports the dearest."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "test-key",
|
||||
"input_cost_per_token": 9e-06,
|
||||
"output_cost_per_token": 9e-05,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "test-key",
|
||||
"input_cost_per_token": 4e-05,
|
||||
"output_cost_per_token": 8e-05,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini")
|
||||
|
||||
assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == {
|
||||
(9e-06, 9e-05),
|
||||
(4e-05, 8e-05),
|
||||
}, f"every matched deployment must be returned, got {prices}"
|
||||
|
||||
|
||||
def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced():
|
||||
"""Pattern matching is the expensive path, so an unpriced wildcard must not trigger it.
|
||||
|
||||
With no override the catalog is already the right answer, so there is nothing to find.
|
||||
"""
|
||||
router = litellm.Router(
|
||||
model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}}]
|
||||
)
|
||||
|
||||
with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed:
|
||||
assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == ()
|
||||
|
||||
routed.assert_not_called()
|
||||
|
||||
|
||||
def test_get_model_listing_info_carries_configured_limits():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
|
|
|
|||
16
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
16
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -10832,6 +10832,13 @@ export interface paths {
|
|||
* each model's `model_info.service_tiers` as its service tiers; absent,
|
||||
* the OpenAI shape below is returned
|
||||
* - include_metadata: Include additional metadata in the response with fallback information
|
||||
* - include_pricing: Include a `pricing` object per model, carrying
|
||||
* `input_cost_per_token` and `output_cost_per_token` in USD per token.
|
||||
* Custom pricing configured on a deployment takes precedence over the
|
||||
* built-in cost map. A price the proxy does not know is reported as
|
||||
* `null`, never 0, so an unmapped model is not read as free. When a
|
||||
* model group's deployments disagree on price, the highest is
|
||||
* reported, matching `/model_group/info`.
|
||||
* - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
|
||||
* Defaults to "general" when include_metadata=true
|
||||
* - scope: Optional scope parameter. Currently only accepts "expand".
|
||||
|
|
@ -21542,6 +21549,13 @@ export interface paths {
|
|||
* each model's `model_info.service_tiers` as its service tiers; absent,
|
||||
* the OpenAI shape below is returned
|
||||
* - include_metadata: Include additional metadata in the response with fallback information
|
||||
* - include_pricing: Include a `pricing` object per model, carrying
|
||||
* `input_cost_per_token` and `output_cost_per_token` in USD per token.
|
||||
* Custom pricing configured on a deployment takes precedence over the
|
||||
* built-in cost map. A price the proxy does not know is reported as
|
||||
* `null`, never 0, so an unmapped model is not read as free. When a
|
||||
* model group's deployments disagree on price, the highest is
|
||||
* reported, matching `/model_group/info`.
|
||||
* - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
|
||||
* Defaults to "general" when include_metadata=true
|
||||
* - scope: Optional scope parameter. Currently only accepts "expand".
|
||||
|
|
@ -66618,6 +66632,7 @@ export interface operations {
|
|||
scope?: string | null;
|
||||
healthy_only?: boolean | null;
|
||||
client_version?: string | null;
|
||||
include_pricing?: boolean | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
|
|
@ -80881,6 +80896,7 @@ export interface operations {
|
|||
scope?: string | null;
|
||||
healthy_only?: boolean | null;
|
||||
client_version?: string | null;
|
||||
include_pricing?: boolean | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue