Merge 3b239626c0 into 231a46e40b

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -897,6 +897,14 @@ def calculate_cache_writing_cost(
 ) -> float:
     """
     Adjust cost of cache creation tokens based on the cache creation token details.
+
+    When a 5m/1h breakdown is present but covers fewer tokens than
+    ``cache_creation_tokens`` (common on streamed Anthropic server-tool
+    requests, where the stream merger keeps the ``message_start`` breakdown
+    while the top-level total grows across iterations), bill the undetailed
+    remainder at the default 5m rate — same rule
+    ``AnthropicConfig._aggregate_cache_creation_token_details`` uses for
+    non-streaming ``usage.iterations``.
     """
     total_cost: float = 0.0
     if cache_creation_token_details is not None:
@@ -908,6 +916,9 @@ def calculate_cache_writing_cost(
         total_cost += (
             cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0
         )
+        detailed: Final = (cache_creation_tokens_5m or 0) + (cache_creation_tokens_1h or 0)
+        undetailed: Final = max(cache_creation_tokens - detailed, 0)
+        total_cost += undetailed * cache_creation_cost
     else:
         total_cost += cache_creation_tokens * cache_creation_cost
     return total_cost

tests/test_litellm/litellm_core_utils/test_calculate_cache_writing_cost.py

@@ -0,0 +1,66 @@
+"""Regression tests for calculate_cache_writing_cost undetailed remainder (#42663)."""
+
+import pytest
+
+from litellm.litellm_core_utils.llm_cost_calc.utils import calculate_cache_writing_cost
+from litellm.types.utils import CacheCreationTokenDetails
+
+
+def test_calculate_cache_writing_cost_bills_undetailed_remainder_at_5m_rate():
+    """
+    Streamed Anthropic server-tool runs keep the message_start 5m/1h breakdown
+    while cache_creation_input_tokens grows across iterations. Pricing must
+    still bill the undetailed remainder at the 5m rate instead of dropping it.
+    """
+    rate_5m = 3.75e-6
+    rate_1h = 6.0e-6
+    details = CacheCreationTokenDetails(
+        ephemeral_5m_input_tokens=31490,
+        ephemeral_1h_input_tokens=0,
+    )
+
+    # Stale breakdown (message_start only) vs full streamed total.
+    cost = calculate_cache_writing_cost(
+        cache_creation_tokens=184457,
+        cache_creation_token_details=details,
+        cache_creation_cost_above_1hr=rate_1h,
+        cache_creation_cost=rate_5m,
+    )
+    assert cost == pytest.approx(184457 * rate_5m)
+
+    # Already-reconciled details (non-streaming aggregate path) must not double-bill.
+    reconciled = CacheCreationTokenDetails(
+        ephemeral_5m_input_tokens=184457,
+        ephemeral_1h_input_tokens=0,
+    )
+    cost_reconciled = calculate_cache_writing_cost(
+        cache_creation_tokens=184457,
+        cache_creation_token_details=reconciled,
+        cache_creation_cost_above_1hr=rate_1h,
+        cache_creation_cost=rate_5m,
+    )
+    assert cost_reconciled == pytest.approx(184457 * rate_5m)
+
+    # Mixed 5m + 1h with an undetailed remainder.
+    mixed = CacheCreationTokenDetails(
+        ephemeral_5m_input_tokens=1000,
+        ephemeral_1h_input_tokens=2000,
+    )
+    cost_mixed = calculate_cache_writing_cost(
+        cache_creation_tokens=5000,
+        cache_creation_token_details=mixed,
+        cache_creation_cost_above_1hr=rate_1h,
+        cache_creation_cost=rate_5m,
+    )
+    assert cost_mixed == pytest.approx(1000 * rate_5m + 2000 * rate_1h + 2000 * rate_5m)
+
+
+def test_calculate_cache_writing_cost_without_details_uses_total():
+    rate_5m = 3.75e-6
+    cost = calculate_cache_writing_cost(
+        cache_creation_tokens=1000,
+        cache_creation_token_details=None,
+        cache_creation_cost_above_1hr=6.0e-6,
+        cache_creation_cost=rate_5m,
+    )
+    assert cost == pytest.approx(1000 * rate_5m)