diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index fbcf9c78d3e..7e00879afb4 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -283,11 +283,16 @@ def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bo Gemini entry for cache writes, would carry the whole prompt for nothing and turn a profitable route into a reported loss. Such a model pays its plain input rate for those tokens, so the buckets it cannot price become ordinary input below. + + An explicit ``0.0`` is a published rate, not a missing one, and only the read leg can + take it literally: free reads are real and are the largest discount on offer, while no + provider gives writes away, so a zero there is still pricing nobody published. """ if baseline_info is None: return True, True - return bool(baseline_info.get("cache_read_input_token_cost")), bool( - baseline_info.get("cache_creation_input_token_cost") + return ( + baseline_info.get("cache_read_input_token_cost") is not None, + bool(baseline_info.get("cache_creation_input_token_cost")), ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index de413a86521..a3e72b9036c 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -719,6 +719,73 @@ def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): assert reported == pytest.approx(baseline_pays_input - actually_paid) +def _chat_model_with_free_cache_reads() -> tuple[str, str, str]: + """A chat model the bundled map prices per token but serves cache reads for nothing, + derived from the map rather than hardcoded for the same reason its sibling above is: + a fixed pick goes stale the moment the registry starts pricing that model's reads. + Candidates go through the savings module's own resolver, so the pick is one the code + under test can actually price.""" + for key in sorted(litellm.model_cost): + entry = litellm.model_cost[key] + if entry.get("mode") != "chat" or entry.get("cache_read_input_token_cost") != 0.0: + continue + if not entry.get("input_cost_per_token") or not entry.get("output_cost_per_token"): + continue + identity = _resolve_model(key, None) + if identity is None: + continue + return key, identity.model, identity.provider + raise AssertionError("the bundled map has no per-token chat model with free cache reads") + + +def test_a_baseline_that_serves_cache_reads_free_is_not_charged_its_input_rate(): + """A `0.0` read rate is a published price, not a missing one. Reading the two as the + same thing moved the baseline's cached tokens onto its input rate, so a turn routed off + such a baseline reported a saving inflated by the whole cached prompt. + """ + baseline_key, baseline_name, baseline_provider = _chat_model_with_free_cache_reads() + continuing = _usage(fresh=0, cached=20_000, written=0, out=1_000) + reported = compute_autorouter_savings( + baseline_model=baseline_key, + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=continuing, + conversation_continuing=True, + ) + + baseline = litellm.get_model_info(baseline_name, baseline_provider) + assert baseline.get("cache_read_input_token_cost") == 0.0, "pick a baseline that serves cache reads for free" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_reads_free = 1_000 * baseline["output_cost_per_token"] + actually_paid = 20_000 * haiku["cache_read_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + assert reported == pytest.approx(baseline_reads_free - actually_paid) + + +def test_a_baseline_with_a_zero_cache_write_price_still_pays_its_input_rate(): + """The write leg must not copy the read leg's literal reading. The two zeros mean + opposite things: no provider gives cache writes away, so a zero there is pricing nobody + published and those tokens stay ordinary input. Taking it literally would carry the + baseline's 20k prompt for free and report a loss on traffic that saved money. + """ + first_turn = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="deepseek/deepseek-chat", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=first_turn, + conversation_continuing=False, + ) + + baseline = litellm.get_model_info("deepseek-chat", "deepseek") + assert baseline.get("cache_creation_input_token_cost") == 0.0, ( + "pick a baseline publishing a literal 0.0 write price" + ) + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * baseline["input_cost_per_token"] + 1_000 * baseline["output_cost_per_token"] + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + assert reported == pytest.approx(baseline_pays_input - actually_paid) + + def _breakdown(input_cost: float, output_cost: float = 0.0, **extra: object) -> dict: """A `cost_breakdown` as the cost calculator records it on the spend log.""" return {"input_cost": input_cost, "output_cost": output_cost, **extra}