diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 950fcca2039..f4ed217a648 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,9 +8,15 @@ are known) and summed into the daily tables; tokens cannot be priced after they have been aggregated across models. """ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Final, NamedTuple +from typing import ( + TYPE_CHECKING, + Final, + NamedTuple, + TypeGuard, # noqa: TID251 # TypeGuard narrows optional ModelInfo after the input-rate check + cast, # noqa: TID251 # model_cost dicts are ModelInfo-shaped after the input-rate check +) import litellm from litellm._logging import verbose_proxy_logger @@ -91,6 +97,180 @@ def _effective_model_info(router: "Router | None", deployment_id: str | None, mo return None +def _has_input_price( + info: ModelInfo | None, +) -> TypeGuard[ModelInfo]: # guard-ok: ModelInfo | None is priced iff input_cost_per_token is present + """True when ``info`` carries a real input rate, including an explicit ``0``. + + Router registers two keys per deployment: the deployment id keeps custom prices, + and the shared ``{provider}/{model}`` key has every cost field stripped so two + deployments of the same backend cannot clobber each other. ``get_model_info`` + prefers that shared key, so a custom model absent from the built-in map resolves + to a dict that looks like a hit and then prices at ``0.0``. Treating a missing + rate as "no info" lets the caller keep looking at the deployment id. + """ + return info is not None and info.get("input_cost_per_token") is not None + + +def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]: + """ + Return ``(input_cost, cache_read_cost, cache_write_cost)`` per token. + + ``info`` is whatever pricing the caller resolved -- deployment rates when the + request came through a router deployment, public rates otherwise -- so a + negotiated price is honoured here rather than silently replaced by the list rate. + ``None`` falls open to ``(0.0, 0.0, 0.0)`` so savings degrade to zero rather than + raising inside the spend writer. + + Prices are read through ``_get_cost_per_unit``, the same accessor the cost + calculator uses, which coerces the string prices a ``config.yaml`` can produce + (``"3e-7"``) and resolves service-tier suffixes. + + An absent cache price mirrors the input cost, which yields a zero discount on the + read leg and a zero premium on the write leg. Mirroring rather than taking + ``_get_cost_per_unit``'s 0.0 default is load-bearing on the write leg: a zero write + price would make the premium ``0 - input_cost``, turning a model that simply has no + write pricing into a spurious extra saving. + + The two legs then differ on an explicit ``0.0``, and the asymmetry is deliberate. A + free cache *write* does not exist -- entries carrying a literal zero (``deepseek-chat`` + does) mean "no separate price", so a falsy write price also mirrors input. A free + cache *read* is real: 15 models charge for input and serve reads for nothing, which + is the largest discount available, so the read leg keeps its literal zero. + """ + if info is None: + return 0.0, 0.0, 0.0 + input_cost: Final = _get_cost_per_unit(info, "input_cost_per_token") or 0.0 + cache_read_cost: Final = _get_cost_per_unit(info, "cache_read_input_token_cost", default_value=None) + cache_write_cost: Final = _get_cost_per_unit(info, "cache_creation_input_token_cost", default_value=None) + return ( + input_cost, + input_cost if cache_read_cost is None else cache_read_cost, + cache_write_cost if cache_write_cost else input_cost, + ) + + +def _savings_rate_fingerprint(info: ModelInfo) -> tuple[float, float, float]: + """The effective rates savings uses, after applying cache-price defaults.""" + return _input_cache_read_and_write_cost(info) + + +def _input_rate_fingerprint(info: ModelInfo) -> float: + """The input rate only. Compression does not care about cache prices.""" + input_cost, _, _ = _savings_rate_fingerprint(info) + return input_cost + + +def _cost_map_deployment_info(deployment_id: str | None) -> ModelInfo | None: + """Pricing registered under a deployment id, without needing a live Router. + + Daily spend writes pass ``model_id`` but look the router up lazily. When that + lookup returns ``None``, falling through to the stripped shared key reports + ``$0.00`` savings even though the deployment's rate is already in ``model_cost``. + """ + if not deployment_id: + return None + info = litellm.model_cost.get(deployment_id) + if isinstance(info, dict) and info.get("input_cost_per_token") is not None: + return cast( # cast-ok: model_cost dict already checked for input_cost_per_token; same shape router uses + ModelInfo, info + ) + return None + + +def _deployment_matches_logged_model( + dep_model: str, + dep_provider: object, + public_name: object, + identity: _ModelIdentity | None, + logged_model: str, +) -> bool: + """True when this router row is the deployment a spend log row is talking about.""" + if logged_model and logged_model == public_name: + return True + dep_identity = _resolve_model(dep_model, dep_provider if isinstance(dep_provider, str) else None) + if identity is not None and dep_identity is not None: + return identity == dep_identity + return bool(logged_model) and logged_model == dep_model + + +def _matching_priced_deployments( + router: "Router | None", + identity: _ModelIdentity | None, + logged_model: str, +) -> tuple[ModelInfo, ...]: + """Priced router rows that could be this spend log, or empty if we cannot tell.""" + if router is None: + return () + try: + deployments = router.get_model_list() or () + except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write + verbose_proxy_logger.debug("savings: cannot list deployments (%s)", e) + return () + + priced: tuple[ModelInfo, ...] = () + seen_ids: tuple[str, ...] = () + for dep in deployments: + raw_info = dep.get("model_info") + dep_id = raw_info.get("id") if isinstance(raw_info, dict) else None + if not isinstance(dep_id, str) or dep_id in seen_ids: + continue + raw_params = dep["litellm_params"] + dep_model = str(raw_params.get("model") or "") + dep_provider = raw_params.get("custom_llm_provider") + if not _deployment_matches_logged_model( + dep_model=dep_model, + dep_provider=dep_provider, + public_name=dep.get("model_name"), + identity=identity, + logged_model=logged_model, + ): + continue + seen_ids = (*seen_ids, dep_id) + info = _cost_map_deployment_info(dep_id) or _effective_model_info(router, dep_id, logged_model) + if _has_input_price(info): + priced = (*priced, info) + return priced + + +def _unique_by_rate( + priced: Sequence[ModelInfo], + rate_key: Callable[[ModelInfo], object], +) -> ModelInfo | None: + if not priced: + return None + fingerprints = frozenset(rate_key(info) for info in priced) + return priced[0] if len(fingerprints) == 1 else None + + +def _pricing_for_savings( + router: "Router | None", + model_id: str | None, + identity: _ModelIdentity | None, + model: str | None, + rate_key: Callable[[ModelInfo], object] = _savings_rate_fingerprint, +) -> ModelInfo | None: + """Deployment rate first, public rate only when it actually has a price.""" + logged: Final = model or "" + priced: Final = () if model_id else _matching_priced_deployments(router, identity, logged) + unique: Final = _unique_by_rate(priced, rate_key) if priced else None + if priced and unique is None: + # Matching deployments disagree. The public list price is not a + # substitute — it can match none of them. + return None + extra: Final = (unique,) if unique is not None else () + candidates: Final = ( + _cost_map_deployment_info(model_id), + _effective_model_info(router, model_id, logged), + *extra, + _model_info(identity) if identity else None, + ) + for candidate in candidates: + if _has_input_price(candidate): + return candidate + return None + + def _model_info(model: _ModelIdentity) -> ModelInfo | None: """The public rates for ``model``, or ``None`` when it has none.""" try: @@ -616,21 +796,28 @@ def compute_savings_spend( nothing and recompute, mirroring ``_recorded_token_cost``. """ # Deployment rates when the request came through one, public rates otherwise -- - # `_effective_model_info` merges a deployment's configured prices over the built-in - # map, so a negotiated price is not silently replaced by the list rate. + # `_pricing_for_savings` prefers the deployment-id cost-map entry (even with no + # live Router) so a custom model whose shared `{provider}/{model}` key had its + # prices stripped is not silently billed at $0. + # Without model_id, compression can still use a unique input rate when cache + # rates differ; prompt-caching needs the full fingerprint or it would pick + # the first deployment's cache price. router_instance: Router | None = llm_router() if llm_router else None identity: Final = _resolve_model(model, custom_llm_provider) - pricing: Final = _effective_model_info(router_instance, model_id, model or "") or ( - _model_info(identity) if identity else None + cache_pricing: Final = _pricing_for_savings(router_instance, model_id, identity, model) + compression_pricing: Final = ( + cache_pricing + if model_id + else _pricing_for_savings(router_instance, model_id, identity, model, rate_key=_input_rate_fingerprint) ) - input_cost: Final = (_get_cost_per_unit(pricing, "input_cost_per_token") or 0.0) if pricing else 0.0 - compression: Final = max(compression_saved_tokens, 0) * input_cost + compression_input, _, _ = _input_cache_read_and_write_cost(compression_pricing) + compression: Final = max(compression_saved_tokens, 0) * compression_input usage: Final = _usage_from_spend_log(usage_object) basis: Final = _pricing_basis(cost_breakdown) billed_at_datetime: Final = _coerce_billed_at(billed_at) prompt_caching: Final = ( calculate_prompt_caching_savings( - model_info=pricing, + model_info=cache_pricing, usage=usage, custom_llm_provider=identity.provider if identity else custom_llm_provider, service_tier=basis.service_tier, @@ -638,7 +825,7 @@ def compute_savings_spend( vertex_location=basis.vertex_location, billed_at=billed_at_datetime, ) - if pricing is not None and usage is not None + if cache_pricing is not None and usage is not None else 0.0 ) gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index e466edab131..96633b2dbe1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -1302,6 +1302,250 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): assert with_deployment_rate.autorouter > at_public_rate.autorouter +def _custom_unmapped_router(): + """A self-hosted model that is not in the built-in cost map.""" + return Router( + model_list=[ + { + "model_name": "muse-glimmer-30b", + "litellm_params": { + "model": "muse-glimmer-30b", + "custom_llm_provider": "openai", + "api_base": "https://example.invalid/v1", + "api_key": "sk-test", + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 3.5e-08, + }, + }, + ] + ) + + +def test_custom_unmapped_model_compression_savings_use_deployment_id_without_router(): + """A custom model's rate lives on the deployment-id cost-map key. + + The spend writer always has ``model_id``; it does not always have a live Router. + Looking the rate up only through ``get_model_info`` hits the stripped shared + backend key and reports $0.00 next to a non-zero token count. + """ + router = _custom_unmapped_router() + deployment_id = router.get_model_list(model_name="muse-glimmer-30b")[0]["model_info"]["id"] + + result = compute_savings_spend( + model="muse-glimmer-30b", + custom_llm_provider="openai", + compression_saved_tokens=2642007, + gateway_injected_cache=False, + model_id=deployment_id, + ) + assert result.compression == pytest.approx(2642007 * 3.5e-07) + assert result.compression > 0 + + +def test_custom_unmapped_model_compression_savings_without_model_id_use_unique_deployment(): + """A single custom deployment can still be priced when the log omitted model_id.""" + router = _custom_unmapped_router() + result = compute_savings_spend( + model="muse-glimmer-30b", + custom_llm_provider="openai", + compression_saved_tokens=100000, + gateway_injected_cache=False, + llm_router=lambda: router, + ) + assert result.compression == pytest.approx(100000 * 3.5e-07) + + +def test_custom_unmapped_model_prompt_caching_savings_use_deployment_rate(): + router = _custom_unmapped_router() + deployment_id = router.get_model_list(model_name="muse-glimmer-30b")[0]["model_info"]["id"] + result = compute_savings_spend( + model="muse-glimmer-30b", + custom_llm_provider="openai", + compression_saved_tokens=0, + gateway_injected_cache=False, + usage_object={"cache_read_input_tokens": 500000}, + model_id=deployment_id, + ) + assert result.prompt_caching == pytest.approx(500000 * (3.5e-07 - 3.5e-08)) + assert result.prompt_caching > 0 + + +def test_two_custom_deployments_at_different_rates_need_model_id(): + """The public model_name is shared; without model_id the rate cannot be guessed.""" + router = Router( + model_list=[ + { + "model_name": "deepseek-v4-pro", + "litellm_params": { + "model": "openrouter/deepseek/deepseek-v4-pro", + "api_key": "sk-openrouter", + "input_cost_per_token": 4.225e-07, + "output_cost_per_token": 8.45e-07, + "cache_read_input_token_cost": 3.5e-08, + }, + }, + { + "model_name": "deepseek-v4-pro", + "litellm_params": { + "model": "openai/deepseek-v4-pro", + "custom_llm_provider": "openai", + "api_base": "https://example.invalid/zen", + "api_key": "sk-zen", + "input_cost_per_token": 1.74e-06, + "output_cost_per_token": 3.84e-06, + "cache_read_input_token_cost": 1.74e-07, + }, + }, + ] + ) + openrouter_id = router.get_model_list(model_name="deepseek-v4-pro")[0]["model_info"]["id"] + without_id = compute_savings_spend( + model="deepseek-v4-pro", + custom_llm_provider="openai", + compression_saved_tokens=100000, + gateway_injected_cache=False, + llm_router=lambda: router, + ) + with_id = compute_savings_spend( + model="deepseek/deepseek-v4-pro", + custom_llm_provider="openrouter", + compression_saved_tokens=100000, + gateway_injected_cache=False, + model_id=openrouter_id, + ) + assert without_id.compression == 0.0 + assert with_id.compression == pytest.approx(100000 * 4.225e-07) + + +def test_same_input_rate_different_cache_rates_still_price_compression(): + """Compression only needs the input rate; prompt-caching still needs model_id. + + Prompt-caching savings are ``tokens * (input - cache_read)``. Two deployments + at the same input price but different cache-read prices cannot share a cache + rate, but compression can still be priced from the unique input rate. + """ + router = Router( + model_list=[ + { + "model_name": "muse-glimmer-30b", + "litellm_params": { + "model": "muse-glimmer-30b", + "custom_llm_provider": "openai", + "api_base": "https://example.invalid/v1", + "api_key": "sk-a", + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 3.5e-08, + }, + }, + { + "model_name": "muse-glimmer-30b", + "litellm_params": { + "model": "muse-glimmer-30b", + "custom_llm_provider": "openai", + "api_base": "https://example.invalid/v2", + "api_key": "sk-b", + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.0e-07, + }, + }, + ] + ) + without_id = compute_savings_spend( + model="muse-glimmer-30b", + custom_llm_provider="openai", + compression_saved_tokens=100000, + gateway_injected_cache=False, + usage_object={"cache_read_input_tokens": 500000}, + llm_router=lambda: router, + ) + assert without_id.compression == pytest.approx(100000 * 3.5e-07) + assert without_id.prompt_caching == 0.0 + + +def test_omitted_cache_rate_matches_explicit_mirror_of_input(): + """A missing cache price is the same effective rate as cache_read == input.""" + router = Router( + model_list=[ + { + "model_name": "muse-glimmer-30b", + "litellm_params": { + "model": "muse-glimmer-30b", + "custom_llm_provider": "openai", + "api_base": "https://example.invalid/v1", + "api_key": "sk-a", + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 1.5e-06, + }, + }, + { + "model_name": "muse-glimmer-30b", + "litellm_params": { + "model": "muse-glimmer-30b", + "custom_llm_provider": "openai", + "api_base": "https://example.invalid/v2", + "api_key": "sk-b", + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 3.5e-07, + "cache_creation_input_token_cost": 3.5e-07, + }, + }, + ] + ) + result = compute_savings_spend( + model="muse-glimmer-30b", + custom_llm_provider="openai", + compression_saved_tokens=100000, + gateway_injected_cache=False, + llm_router=lambda: router, + ) + assert result.compression == pytest.approx(100000 * 3.5e-07) + + +def test_ambiguous_cache_rates_do_not_fall_back_to_public_prompt_caching(): + """Disagreeing deployment cache rates must not be replaced by the public map.""" + public_input, public_cache_read = _anthropic_costs("claude-sonnet-5") + router = Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": { + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "api_key": "sk-a", + "input_cost_per_token": public_input, + "cache_read_input_token_cost": 1.0e-08, + }, + }, + { + "model_name": "claude-sonnet-5", + "litellm_params": { + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "api_key": "sk-b", + "input_cost_per_token": public_input, + "cache_read_input_token_cost": 5.0e-08, + }, + }, + ] + ) + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=100000, + gateway_injected_cache=False, + usage_object={"cache_read_input_tokens": 500000}, + llm_router=lambda: router, + ) + public_prompt_caching = 500000 * max(public_input - public_cache_read, 0.0) + assert result.compression == pytest.approx(100000 * public_input) + assert result.prompt_caching == 0.0 + assert public_prompt_caching != 0.0 + + def _routed_decision() -> dict: return {"savings_baseline_model": "anthropic/claude-opus-5", "conversation_continuing": True}