Merge 8c09f00738 into c2c2a623c0

litellm/proxy/spend_tracking/savings.py

@@ -8,9 +8,15 @@ are known) and summed into the daily tables; tokens cannot be priced after they
 have been aggregated across models.
 """
 
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime
-from typing import TYPE_CHECKING, Final, NamedTuple
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    NamedTuple,
+    TypeGuard,  # noqa: TID251  # TypeGuard narrows optional ModelInfo after the input-rate check
+    cast,  # noqa: TID251  # model_cost dicts are ModelInfo-shaped after the input-rate check
+)
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -91,6 +97,180 @@ def _effective_model_info(router: "Router | None", deployment_id: str | None, mo
         return None
 
 
+def _has_input_price(
+    info: ModelInfo | None,
+) -> TypeGuard[ModelInfo]:  # guard-ok: ModelInfo | None is priced iff input_cost_per_token is present
+    """True when ``info`` carries a real input rate, including an explicit ``0``.
+
+    Router registers two keys per deployment: the deployment id keeps custom prices,
+    and the shared ``{provider}/{model}`` key has every cost field stripped so two
+    deployments of the same backend cannot clobber each other. ``get_model_info``
+    prefers that shared key, so a custom model absent from the built-in map resolves
+    to a dict that looks like a hit and then prices at ``0.0``. Treating a missing
+    rate as "no info" lets the caller keep looking at the deployment id.
+    """
+    return info is not None and info.get("input_cost_per_token") is not None
+
+
+def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]:
+    """
+    Return ``(input_cost, cache_read_cost, cache_write_cost)`` per token.
+
+    ``info`` is whatever pricing the caller resolved -- deployment rates when the
+    request came through a router deployment, public rates otherwise -- so a
+    negotiated price is honoured here rather than silently replaced by the list rate.
+    ``None`` falls open to ``(0.0, 0.0, 0.0)`` so savings degrade to zero rather than
+    raising inside the spend writer.
+
+    Prices are read through ``_get_cost_per_unit``, the same accessor the cost
+    calculator uses, which coerces the string prices a ``config.yaml`` can produce
+    (``"3e-7"``) and resolves service-tier suffixes.
+
+    An absent cache price mirrors the input cost, which yields a zero discount on the
+    read leg and a zero premium on the write leg. Mirroring rather than taking
+    ``_get_cost_per_unit``'s 0.0 default is load-bearing on the write leg: a zero write
+    price would make the premium ``0 - input_cost``, turning a model that simply has no
+    write pricing into a spurious extra saving.
+
+    The two legs then differ on an explicit ``0.0``, and the asymmetry is deliberate. A
+    free cache *write* does not exist -- entries carrying a literal zero (``deepseek-chat``
+    does) mean "no separate price", so a falsy write price also mirrors input. A free
+    cache *read* is real: 15 models charge for input and serve reads for nothing, which
+    is the largest discount available, so the read leg keeps its literal zero.
+    """
+    if info is None:
+        return 0.0, 0.0, 0.0
+    input_cost: Final = _get_cost_per_unit(info, "input_cost_per_token") or 0.0
+    cache_read_cost: Final = _get_cost_per_unit(info, "cache_read_input_token_cost", default_value=None)
+    cache_write_cost: Final = _get_cost_per_unit(info, "cache_creation_input_token_cost", default_value=None)
+    return (
+        input_cost,
+        input_cost if cache_read_cost is None else cache_read_cost,
+        cache_write_cost if cache_write_cost else input_cost,
+    )
+
+
+def _savings_rate_fingerprint(info: ModelInfo) -> tuple[float, float, float]:
+    """The effective rates savings uses, after applying cache-price defaults."""
+    return _input_cache_read_and_write_cost(info)
+
+
+def _input_rate_fingerprint(info: ModelInfo) -> float:
+    """The input rate only. Compression does not care about cache prices."""
+    input_cost, _, _ = _savings_rate_fingerprint(info)
+    return input_cost
+
+
+def _cost_map_deployment_info(deployment_id: str | None) -> ModelInfo | None:
+    """Pricing registered under a deployment id, without needing a live Router.
+
+    Daily spend writes pass ``model_id`` but look the router up lazily. When that
+    lookup returns ``None``, falling through to the stripped shared key reports
+    ``$0.00`` savings even though the deployment's rate is already in ``model_cost``.
+    """
+    if not deployment_id:
+        return None
+    info = litellm.model_cost.get(deployment_id)
+    if isinstance(info, dict) and info.get("input_cost_per_token") is not None:
+        return cast(  # cast-ok: model_cost dict already checked for input_cost_per_token; same shape router uses
+            ModelInfo, info
+        )
+    return None
+
+
+def _deployment_matches_logged_model(
+    dep_model: str,
+    dep_provider: object,
+    public_name: object,
+    identity: _ModelIdentity | None,
+    logged_model: str,
+) -> bool:
+    """True when this router row is the deployment a spend log row is talking about."""
+    if logged_model and logged_model == public_name:
+        return True
+    dep_identity = _resolve_model(dep_model, dep_provider if isinstance(dep_provider, str) else None)
+    if identity is not None and dep_identity is not None:
+        return identity == dep_identity
+    return bool(logged_model) and logged_model == dep_model
+
+
+def _matching_priced_deployments(
+    router: "Router | None",
+    identity: _ModelIdentity | None,
+    logged_model: str,
+) -> tuple[ModelInfo, ...]:
+    """Priced router rows that could be this spend log, or empty if we cannot tell."""
+    if router is None:
+        return ()
+    try:
+        deployments = router.get_model_list() or ()
+    except Exception as e:  # noqa: BLE001  # a dashboard metric must not fail the spend write
+        verbose_proxy_logger.debug("savings: cannot list deployments (%s)", e)
+        return ()
+
+    priced: tuple[ModelInfo, ...] = ()
+    seen_ids: tuple[str, ...] = ()
+    for dep in deployments:
+        raw_info = dep.get("model_info")
+        dep_id = raw_info.get("id") if isinstance(raw_info, dict) else None
+        if not isinstance(dep_id, str) or dep_id in seen_ids:
+            continue
+        raw_params = dep["litellm_params"]
+        dep_model = str(raw_params.get("model") or "")
+        dep_provider = raw_params.get("custom_llm_provider")
+        if not _deployment_matches_logged_model(
+            dep_model=dep_model,
+            dep_provider=dep_provider,
+            public_name=dep.get("model_name"),
+            identity=identity,
+            logged_model=logged_model,
+        ):
+            continue
+        seen_ids = (*seen_ids, dep_id)
+        info = _cost_map_deployment_info(dep_id) or _effective_model_info(router, dep_id, logged_model)
+        if _has_input_price(info):
+            priced = (*priced, info)
+    return priced
+
+
+def _unique_by_rate(
+    priced: Sequence[ModelInfo],
+    rate_key: Callable[[ModelInfo], object],
+) -> ModelInfo | None:
+    if not priced:
+        return None
+    fingerprints = frozenset(rate_key(info) for info in priced)
+    return priced[0] if len(fingerprints) == 1 else None
+
+
+def _pricing_for_savings(
+    router: "Router | None",
+    model_id: str | None,
+    identity: _ModelIdentity | None,
+    model: str | None,
+    rate_key: Callable[[ModelInfo], object] = _savings_rate_fingerprint,
+) -> ModelInfo | None:
+    """Deployment rate first, public rate only when it actually has a price."""
+    logged: Final = model or ""
+    priced: Final = () if model_id else _matching_priced_deployments(router, identity, logged)
+    unique: Final = _unique_by_rate(priced, rate_key) if priced else None
+    if priced and unique is None:
+        # Matching deployments disagree. The public list price is not a
+        # substitute — it can match none of them.
+        return None
+    extra: Final = (unique,) if unique is not None else ()
+    candidates: Final = (
+        _cost_map_deployment_info(model_id),
+        _effective_model_info(router, model_id, logged),
+        *extra,
+        _model_info(identity) if identity else None,
+    )
+    for candidate in candidates:
+        if _has_input_price(candidate):
+            return candidate
+    return None
+
+
 def _model_info(model: _ModelIdentity) -> ModelInfo | None:
     """The public rates for ``model``, or ``None`` when it has none."""
     try:
@@ -616,21 +796,28 @@ def compute_savings_spend(
     nothing and recompute, mirroring ``_recorded_token_cost``.
     """
     # Deployment rates when the request came through one, public rates otherwise --
-    # `_effective_model_info` merges a deployment's configured prices over the built-in
-    # map, so a negotiated price is not silently replaced by the list rate.
+    # `_pricing_for_savings` prefers the deployment-id cost-map entry (even with no
+    # live Router) so a custom model whose shared `{provider}/{model}` key had its
+    # prices stripped is not silently billed at $0.
+    # Without model_id, compression can still use a unique input rate when cache
+    # rates differ; prompt-caching needs the full fingerprint or it would pick
+    # the first deployment's cache price.
     router_instance: Router | None = llm_router() if llm_router else None
     identity: Final = _resolve_model(model, custom_llm_provider)
-    pricing: Final = _effective_model_info(router_instance, model_id, model or "") or (
-        _model_info(identity) if identity else None
+    cache_pricing: Final = _pricing_for_savings(router_instance, model_id, identity, model)
+    compression_pricing: Final = (
+        cache_pricing
+        if model_id
+        else _pricing_for_savings(router_instance, model_id, identity, model, rate_key=_input_rate_fingerprint)
     )
-    input_cost: Final = (_get_cost_per_unit(pricing, "input_cost_per_token") or 0.0) if pricing else 0.0
-    compression: Final = max(compression_saved_tokens, 0) * input_cost
+    compression_input, _, _ = _input_cache_read_and_write_cost(compression_pricing)
+    compression: Final = max(compression_saved_tokens, 0) * compression_input
     usage: Final = _usage_from_spend_log(usage_object)
     basis: Final = _pricing_basis(cost_breakdown)
     billed_at_datetime: Final = _coerce_billed_at(billed_at)
     prompt_caching: Final = (
         calculate_prompt_caching_savings(
-            model_info=pricing,
+            model_info=cache_pricing,
             usage=usage,
             custom_llm_provider=identity.provider if identity else custom_llm_provider,
             service_tier=basis.service_tier,
@@ -638,7 +825,7 @@ def compute_savings_spend(
             vertex_location=basis.vertex_location,
             billed_at=billed_at_datetime,
         )
-        if pricing is not None and usage is not None
+        if cache_pricing is not None and usage is not None
         else 0.0
     )
     gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0

tests/test_litellm/proxy/spend_tracking/test_savings.py

@@ -1302,6 +1302,250 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
     assert with_deployment_rate.autorouter > at_public_rate.autorouter
 
 
+def _custom_unmapped_router():
+    """A self-hosted model that is not in the built-in cost map."""
+    return Router(
+        model_list=[
+            {
+                "model_name": "muse-glimmer-30b",
+                "litellm_params": {
+                    "model": "muse-glimmer-30b",
+                    "custom_llm_provider": "openai",
+                    "api_base": "https://example.invalid/v1",
+                    "api_key": "sk-test",
+                    "input_cost_per_token": 3.5e-07,
+                    "output_cost_per_token": 1.5e-06,
+                    "cache_read_input_token_cost": 3.5e-08,
+                },
+            },
+        ]
+    )
+
+
+def test_custom_unmapped_model_compression_savings_use_deployment_id_without_router():
+    """A custom model's rate lives on the deployment-id cost-map key.
+
+    The spend writer always has ``model_id``; it does not always have a live Router.
+    Looking the rate up only through ``get_model_info`` hits the stripped shared
+    backend key and reports $0.00 next to a non-zero token count.
+    """
+    router = _custom_unmapped_router()
+    deployment_id = router.get_model_list(model_name="muse-glimmer-30b")[0]["model_info"]["id"]
+
+    result = compute_savings_spend(
+        model="muse-glimmer-30b",
+        custom_llm_provider="openai",
+        compression_saved_tokens=2642007,
+        gateway_injected_cache=False,
+        model_id=deployment_id,
+    )
+    assert result.compression == pytest.approx(2642007 * 3.5e-07)
+    assert result.compression > 0
+
+
+def test_custom_unmapped_model_compression_savings_without_model_id_use_unique_deployment():
+    """A single custom deployment can still be priced when the log omitted model_id."""
+    router = _custom_unmapped_router()
+    result = compute_savings_spend(
+        model="muse-glimmer-30b",
+        custom_llm_provider="openai",
+        compression_saved_tokens=100000,
+        gateway_injected_cache=False,
+        llm_router=lambda: router,
+    )
+    assert result.compression == pytest.approx(100000 * 3.5e-07)
+
+
+def test_custom_unmapped_model_prompt_caching_savings_use_deployment_rate():
+    router = _custom_unmapped_router()
+    deployment_id = router.get_model_list(model_name="muse-glimmer-30b")[0]["model_info"]["id"]
+    result = compute_savings_spend(
+        model="muse-glimmer-30b",
+        custom_llm_provider="openai",
+        compression_saved_tokens=0,
+        gateway_injected_cache=False,
+        usage_object={"cache_read_input_tokens": 500000},
+        model_id=deployment_id,
+    )
+    assert result.prompt_caching == pytest.approx(500000 * (3.5e-07 - 3.5e-08))
+    assert result.prompt_caching > 0
+
+
+def test_two_custom_deployments_at_different_rates_need_model_id():
+    """The public model_name is shared; without model_id the rate cannot be guessed."""
+    router = Router(
+        model_list=[
+            {
+                "model_name": "deepseek-v4-pro",
+                "litellm_params": {
+                    "model": "openrouter/deepseek/deepseek-v4-pro",
+                    "api_key": "sk-openrouter",
+                    "input_cost_per_token": 4.225e-07,
+                    "output_cost_per_token": 8.45e-07,
+                    "cache_read_input_token_cost": 3.5e-08,
+                },
+            },
+            {
+                "model_name": "deepseek-v4-pro",
+                "litellm_params": {
+                    "model": "openai/deepseek-v4-pro",
+                    "custom_llm_provider": "openai",
+                    "api_base": "https://example.invalid/zen",
+                    "api_key": "sk-zen",
+                    "input_cost_per_token": 1.74e-06,
+                    "output_cost_per_token": 3.84e-06,
+                    "cache_read_input_token_cost": 1.74e-07,
+                },
+            },
+        ]
+    )
+    openrouter_id = router.get_model_list(model_name="deepseek-v4-pro")[0]["model_info"]["id"]
+    without_id = compute_savings_spend(
+        model="deepseek-v4-pro",
+        custom_llm_provider="openai",
+        compression_saved_tokens=100000,
+        gateway_injected_cache=False,
+        llm_router=lambda: router,
+    )
+    with_id = compute_savings_spend(
+        model="deepseek/deepseek-v4-pro",
+        custom_llm_provider="openrouter",
+        compression_saved_tokens=100000,
+        gateway_injected_cache=False,
+        model_id=openrouter_id,
+    )
+    assert without_id.compression == 0.0
+    assert with_id.compression == pytest.approx(100000 * 4.225e-07)
+
+
+def test_same_input_rate_different_cache_rates_still_price_compression():
+    """Compression only needs the input rate; prompt-caching still needs model_id.
+
+    Prompt-caching savings are ``tokens * (input - cache_read)``. Two deployments
+    at the same input price but different cache-read prices cannot share a cache
+    rate, but compression can still be priced from the unique input rate.
+    """
+    router = Router(
+        model_list=[
+            {
+                "model_name": "muse-glimmer-30b",
+                "litellm_params": {
+                    "model": "muse-glimmer-30b",
+                    "custom_llm_provider": "openai",
+                    "api_base": "https://example.invalid/v1",
+                    "api_key": "sk-a",
+                    "input_cost_per_token": 3.5e-07,
+                    "output_cost_per_token": 1.5e-06,
+                    "cache_read_input_token_cost": 3.5e-08,
+                },
+            },
+            {
+                "model_name": "muse-glimmer-30b",
+                "litellm_params": {
+                    "model": "muse-glimmer-30b",
+                    "custom_llm_provider": "openai",
+                    "api_base": "https://example.invalid/v2",
+                    "api_key": "sk-b",
+                    "input_cost_per_token": 3.5e-07,
+                    "output_cost_per_token": 1.5e-06,
+                    "cache_read_input_token_cost": 1.0e-07,
+                },
+            },
+        ]
+    )
+    without_id = compute_savings_spend(
+        model="muse-glimmer-30b",
+        custom_llm_provider="openai",
+        compression_saved_tokens=100000,
+        gateway_injected_cache=False,
+        usage_object={"cache_read_input_tokens": 500000},
+        llm_router=lambda: router,
+    )
+    assert without_id.compression == pytest.approx(100000 * 3.5e-07)
+    assert without_id.prompt_caching == 0.0
+
+
+def test_omitted_cache_rate_matches_explicit_mirror_of_input():
+    """A missing cache price is the same effective rate as cache_read == input."""
+    router = Router(
+        model_list=[
+            {
+                "model_name": "muse-glimmer-30b",
+                "litellm_params": {
+                    "model": "muse-glimmer-30b",
+                    "custom_llm_provider": "openai",
+                    "api_base": "https://example.invalid/v1",
+                    "api_key": "sk-a",
+                    "input_cost_per_token": 3.5e-07,
+                    "output_cost_per_token": 1.5e-06,
+                },
+            },
+            {
+                "model_name": "muse-glimmer-30b",
+                "litellm_params": {
+                    "model": "muse-glimmer-30b",
+                    "custom_llm_provider": "openai",
+                    "api_base": "https://example.invalid/v2",
+                    "api_key": "sk-b",
+                    "input_cost_per_token": 3.5e-07,
+                    "output_cost_per_token": 1.5e-06,
+                    "cache_read_input_token_cost": 3.5e-07,
+                    "cache_creation_input_token_cost": 3.5e-07,
+                },
+            },
+        ]
+    )
+    result = compute_savings_spend(
+        model="muse-glimmer-30b",
+        custom_llm_provider="openai",
+        compression_saved_tokens=100000,
+        gateway_injected_cache=False,
+        llm_router=lambda: router,
+    )
+    assert result.compression == pytest.approx(100000 * 3.5e-07)
+
+
+def test_ambiguous_cache_rates_do_not_fall_back_to_public_prompt_caching():
+    """Disagreeing deployment cache rates must not be replaced by the public map."""
+    public_input, public_cache_read = _anthropic_costs("claude-sonnet-5")
+    router = Router(
+        model_list=[
+            {
+                "model_name": "claude-sonnet-5",
+                "litellm_params": {
+                    "model": "claude-sonnet-5",
+                    "custom_llm_provider": "anthropic",
+                    "api_key": "sk-a",
+                    "input_cost_per_token": public_input,
+                    "cache_read_input_token_cost": 1.0e-08,
+                },
+            },
+            {
+                "model_name": "claude-sonnet-5",
+                "litellm_params": {
+                    "model": "claude-sonnet-5",
+                    "custom_llm_provider": "anthropic",
+                    "api_key": "sk-b",
+                    "input_cost_per_token": public_input,
+                    "cache_read_input_token_cost": 5.0e-08,
+                },
+            },
+        ]
+    )
+    result = compute_savings_spend(
+        model="claude-sonnet-5",
+        custom_llm_provider="anthropic",
+        compression_saved_tokens=100000,
+        gateway_injected_cache=False,
+        usage_object={"cache_read_input_tokens": 500000},
+        llm_router=lambda: router,
+    )
+    public_prompt_caching = 500000 * max(public_input - public_cache_read, 0.0)
+    assert result.compression == pytest.approx(100000 * public_input)
+    assert result.prompt_caching == 0.0
+    assert public_prompt_caching != 0.0
+
+
 def _routed_decision() -> dict:
     return {"savings_baseline_model": "anthropic/claude-opus-5", "conversation_continuing": True}