diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index fbcf9c78d3e..4776ae729ff 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -283,12 +283,29 @@ def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bo Gemini entry for cache writes, would carry the whole prompt for nothing and turn a profitable route into a reported loss. Such a model pays its plain input rate for those tokens, so the buckets it cannot price become ordinary input below. + + The two buckets need different tests, because a `0.0` means something different in + each and the cost map proves it. + + Reads: an explicit `0.0` is a real "cache reads are free" price, so absence rather + than truthiness is the right test. It only means that on a model that actually + caches, though. Six entries pair `cache_read_input_token_cost` of `0` with + `supports_prompt_caching` of `False`, `gemini-robotics-er-1.5-preview` and + `openrouter/z-ai/glm-4.7` among them, where the zero is a placeholder for a model + that has no cache rather than a free one. Honouring it priced 20,000 baseline tokens + at `$0.00` instead of `$0.006`. + + Writes: truthiness stays. A `0.0` cache-creation price is not free, it means writes + bill at the plain input rate, and 36 entries rely on that inheritance including + `deepseek/deepseek-chat`. Reading it as a real price dropped a 10,000 token + first-turn baseline from `$0.0028` to `$0.00`. """ if baseline_info is None: return True, True - return bool(baseline_info.get("cache_read_input_token_cost")), bool( - baseline_info.get("cache_creation_input_token_cost") + prices_reads: Final = baseline_info.get("cache_read_input_token_cost") is not None and bool( + baseline_info.get("supports_prompt_caching") ) + return prices_reads, bool(baseline_info.get("cache_creation_input_token_cost")) def _baseline_usage(usage: Usage, baseline_info: ModelInfo | None = None) -> Usage: diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index de413a86521..d10f6ef1892 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -1374,3 +1374,78 @@ def test_modeled_baseline_uses_recorded_prices_and_preserves_other_actual_charge assert result.savings == pytest.approx(-0.09) assert price_baseline_comparison(snapshot.model_copy(update={"prices": None}), usage, "modeled") is None assert marks_gateway_injection({"litellm_gateway_injected_cache": True}, "dep-a") is False + + +def _usage_with_cached(cached: int) -> Usage: + """A continuing-conversation request whose whole prompt came from cache.""" + return Usage( + prompt_tokens=cached, + completion_tokens=0, + total_tokens=cached, + prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": 0, "text_tokens": 0}, + cache_read_input_tokens=cached, + ) + + +def _usage_with_written(written: int) -> Usage: + """A first-turn request whose whole prompt was written to cache.""" + return Usage( + prompt_tokens=written, + completion_tokens=0, + total_tokens=written, + prompt_tokens_details={"cached_tokens": 0, "cache_creation_tokens": written, "text_tokens": 0}, + cache_creation_input_tokens=written, + ) + + +def test_zero_cache_read_rate_is_free_only_when_the_model_actually_caches(): + """A `0.0` read rate is a real price on a caching model and a placeholder otherwise. + + `tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8` prices cache reads at `0` and + sets `supports_prompt_caching`, so its reads really are free and stay in the read + bucket. `gemini/gemini-robotics-er-1.5-preview` carries the same `0` with + `supports_prompt_caching` false: it has no cache at all, so those tokens are its + plain input and must move to `text_tokens`. Reading the zero as a price either way + billed 20,000 baseline tokens at `$0.00` instead of `$0.006`. + """ + caches = litellm.get_model_info(model="tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8") + does_not = litellm.get_model_info(model="gemini/gemini-robotics-er-1.5-preview") + assert caches["cache_read_input_token_cost"] == 0 and caches["supports_prompt_caching"] + assert does_not["cache_read_input_token_cost"] == 0 and not does_not["supports_prompt_caching"] + + free = _baseline_usage(_usage_with_cached(20_000), conversation_continuing=True, baseline_info=caches) + assert free.prompt_tokens_details.cached_tokens == 20_000 + assert (free.prompt_tokens_details.text_tokens or 0) == 0 + + unpriced = _baseline_usage(_usage_with_cached(20_000), conversation_continuing=True, baseline_info=does_not) + assert unpriced.prompt_tokens_details.cached_tokens == 0 + assert unpriced.prompt_tokens_details.text_tokens == 20_000 + # 20,000 at the plain input rate is the $0.006 the buggy read dropped to $0.00. + assert 20_000 * (does_not["input_cost_per_token"] or 0.0) == pytest.approx(0.006) + + +def test_zero_cache_write_rate_still_inherits_input_pricing(): + """A `0.0` cache-creation price means writes bill at input, not that they are free. + + 36 cost-map entries rely on that inheritance, `deepseek/deepseek-chat` among them. + Treating the zero as a real price moved a 10,000 token first-turn baseline from + `$0.0028` to `$0.00`. + """ + info = litellm.get_model_info(model="deepseek/deepseek-chat") + assert info["cache_creation_input_token_cost"] == 0.0 + + baseline = _baseline_usage(_usage_with_written(10_000), conversation_continuing=False, baseline_info=info) + assert (baseline.prompt_tokens_details.cache_creation_tokens or 0) == 0 + assert baseline.prompt_tokens_details.text_tokens == 10_000 + assert 10_000 * (info["input_cost_per_token"] or 0.0) == pytest.approx(0.0028) + + +def test_a_missing_cache_rate_is_still_not_a_free_bucket(): + """The original guard: absent rates fall back to plain input, and `None` info prices both.""" + from litellm.proxy.spend_tracking.savings import _baseline_cache_rate_keys + + assert _baseline_cache_rate_keys({}) == (False, False) + assert _baseline_cache_rate_keys(None) == (True, True) + assert _baseline_cache_rate_keys( + {"cache_read_input_token_cost": 1e-07, "supports_prompt_caching": True, "cache_creation_input_token_cost": 2e-07} + ) == (True, True)