From c2bc5aecff15604f42180d7725c8b34c540bfa11 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:38:08 -0700 Subject: [PATCH] fix(cost_calculator): apply the anthropic fast multiplier to cache cost breakdown (#46112) * fix(cost_calculator): apply the anthropic fast multiplier to cache cost breakdown Anthropic fast mode billed totals at provider_specific_entry.fast while the spend log cost_breakdown kept cache_read_cost and cache_creation_cost at 1x. Move the speed lookup into a shared helper, multiply it into the billed token rates used by get_token_type_cost_breakdown, and have the Anthropic calculator read the same helper so totals and breakdown cannot drift apart. Production change carried from #44074. Co-authored-by: kerry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost_calculator): narrow the provider entry instead of defaulting to a dict literal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry --- .../litellm_core_utils/llm_cost_calc/utils.py | 8 + litellm/llms/anthropic/cost_calculation.py | 9 +- .../cost_calculation/cases/anthropic.py | 315 ++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 31 ++ 4 files changed, 356 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 73428e08e34..2afb6ed4c58 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1269,6 +1269,13 @@ def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> return float(provider_specific_entry.get(inference_geo.lower(), 1.0)) +def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) -> float: + provider_specific_entry: Final = model_info.get("provider_specific_entry") + if getattr(usage, "speed", None) != "fast" or not provider_specific_entry: + return 1.0 + return float(provider_specific_entry.get("fast", 1.0)) + + def _resolve_reasoning_token_cost( model_info: ModelInfo, service_tier: str | None, @@ -1633,6 +1640,7 @@ def _cost_map_billed_rates( get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(model_info, vertex_location) * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) + * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) ) return BilledTokenRates( input_cost_per_token=prompt_base_cost, diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 88805df2b08..94c8149eff3 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -10,6 +10,7 @@ from pydantic import ValidationError from litellm.litellm_core_utils.llm_cost_calc.utils import ( generic_cost_per_token, get_provider_specific_geo_multiplier, + get_provider_specific_speed_multiplier, get_web_search_requests_from_usage, ) from litellm.types.llms.base import LiteLLMBaseModel @@ -50,14 +51,8 @@ def cost_per_token( if model_info is not None else litellm.get_model_info(model=model, custom_llm_provider="anthropic") ) - provider_specific_entry: Final = effective_info.get("provider_specific_entry") - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage) - speed_multiplier: Final = ( - provider_specific_entry.get("fast", 1.0) - if provider_specific_entry and getattr(usage, "speed", None) == "fast" - else 1.0 - ) + speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage) if speed_multiplier != 1.0: prompt_cost *= speed_multiplier diff --git a/tests/integration/cost_calculation/cases/anthropic.py b/tests/integration/cost_calculation/cases/anthropic.py index 762d9469366..3b70ee1e8c5 100644 --- a/tests/integration/cost_calculation/cases/anthropic.py +++ b/tests/integration/cost_calculation/cases/anthropic.py @@ -1459,6 +1459,313 @@ CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE: Final = CostTrackingTestCase( ), ) +CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_CACHE: Final = CostTrackingTestCase( + name="claude-opus-5-anthropic_fast_mode_cache", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-opus-5", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "121e2a9a4452 summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [{"type": "text", "text": "scripted answer 121e2a9a4452"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "cache_read_input_tokens": 400, + "cache_creation_input_tokens": 300, + "cache_creation": {"ephemeral_5m_input_tokens": 100, "ephemeral_1h_input_tokens": 200}, + "speed": "fast", + }, + }, + ), + expected=ExactExpected( + spend=0.13395, + input_cost=0.07215, + output_cost=0.0618, + prompt_tokens=2540, + completion_tokens=412, + cache_read_cost=0.0012, + cache_creation_cost=0.01575, + ), +) + +CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_CACHE_STREAM: Final = CostTrackingTestCase( + name="claude-opus-5-anthropic_fast_mode_cache-stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-opus-5", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6e8ee84cf8b8 summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": True, + "stream_options": {"include_usage": True}, + "allowed_openai_params": [], + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [], + "stop_reason": None, + "usage": { + "input_tokens": 1840, + "cache_read_input_tokens": 400, + "cache_creation_input_tokens": 300, + "cache_creation": {"ephemeral_5m_input_tokens": 100, "ephemeral_1h_input_tokens": 200}, + "speed": "fast", + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "scripted answer 6e8ee84cf8b8"}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 412}}, + {"type": "message_stop"}, + ), + ), + expected=ExactExpected( + spend=0.13395, + input_cost=0.07215, + output_cost=0.0618, + prompt_tokens=2540, + completion_tokens=412, + cache_read_cost=0.0012, + cache_creation_cost=0.01575, + ), +) + +CLAUDE_OPUS_5_MESSAGES_FAST_MODE_CACHE: Final = CostTrackingTestCase( + name="claude-opus-5-messages_fast_mode_cache", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-opus-5", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6d2006a65289 summarize the attached material in one line", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [{"type": "text", "text": "scripted response"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "cache_read_input_tokens": 400, + "cache_creation_input_tokens": 300, + "cache_creation": {"ephemeral_5m_input_tokens": 100, "ephemeral_1h_input_tokens": 200}, + "speed": "fast", + }, + }, + ), + expected=ExactExpected( + spend=0.13395, + input_cost=0.07215, + output_cost=0.0618, + prompt_tokens=2540, + completion_tokens=412, + cache_read_cost=0.0012, + cache_creation_cost=0.01575, + ), +) + +CLAUDE_OPUS_5_ANTHROPIC_STANDARD_SPEED_CACHE: Final = CostTrackingTestCase( + name="claude-opus-5-anthropic_standard_speed_cache", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-opus-5", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2b9904617ede summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [{"type": "text", "text": "scripted answer 2b9904617ede"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "cache_read_input_tokens": 400, + "cache_creation_input_tokens": 300, + "cache_creation": {"ephemeral_5m_input_tokens": 100, "ephemeral_1h_input_tokens": 200}, + "speed": "standard", + }, + }, + ), + expected=ExactExpected( + spend=0.022325, + input_cost=0.012025, + output_cost=0.0103, + prompt_tokens=2540, + completion_tokens=412, + cache_read_cost=0.0002, + cache_creation_cost=0.002625, + ), +) + +CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_US_INFERENCE_CACHE: Final = CostTrackingTestCase( + name="claude-opus-5-anthropic_fast_mode_us_inference_cache", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-opus-5", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "04b13c88ad07 summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [{"type": "text", "text": "scripted answer 04b13c88ad07"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "cache_read_input_tokens": 400, + "cache_creation_input_tokens": 300, + "cache_creation": {"ephemeral_5m_input_tokens": 100, "ephemeral_1h_input_tokens": 200}, + "speed": "fast", + "inference_geo": "us", + }, + }, + ), + expected=ExactExpected( + spend=0.147345, + input_cost=0.079365, + output_cost=0.06798, + prompt_tokens=2540, + completion_tokens=412, + cache_read_cost=0.00132, + cache_creation_cost=0.017325, + ), +) + CLAUDE_OPUS_5_ANTHROPIC_US_INFERENCE: Final = CostTrackingTestCase( name="claude-opus-5-anthropic_us_inference", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -3786,6 +4093,11 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_OPUS_5_TIERED_CACHE_WRITE_ABOVE_200K, CLAUDE_OPUS_5_SERVICE_TIER_PRIORITY, CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE, + CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_CACHE, + CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_CACHE_STREAM, + CLAUDE_OPUS_5_MESSAGES_FAST_MODE_CACHE, + CLAUDE_OPUS_5_ANTHROPIC_STANDARD_SPEED_CACHE, + CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_US_INFERENCE_CACHE, CLAUDE_OPUS_5_ANTHROPIC_US_INFERENCE, CLAUDE_OPUS_5_WEB_SEARCH_MEDIUM, CLAUDE_OPUS_5_STREAM, @@ -3842,6 +4154,9 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=CLAUDE_OPUS_5_RESPONSE_MODEL_OVERRIDE, streamed=CLAUDE_OPUS_5_STREAM_RESPONSE_MODEL_OVERRIDE ), StreamParityTestCase(plain=CLAUDE_OPUS_5_TOOL_CALL, streamed=CLAUDE_OPUS_5_STREAM_TOOL_CALL), + StreamParityTestCase( + plain=CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_CACHE, streamed=CLAUDE_OPUS_5_ANTHROPIC_FAST_MODE_CACHE_STREAM + ), StreamParityTestCase( plain=CLAUDE_SONNET_5_RESPONSE_MODEL_OVERRIDE, streamed=CLAUDE_SONNET_5_STREAM_RESPONSE_MODEL_OVERRIDE ), diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index dbd9cb470e4..be39e83f989 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -3996,3 +3996,34 @@ def test_get_batch_cost_rates_claude_haiku_5_5_prompt_length_tiers( ) assert (rates.input, rates.output, rates.cache_read, rates.cache_creation) == expected + + +@pytest.mark.parametrize(("speed", "multiplier"), (("fast", 6.0), ("standard", 1.0), (None, 1.0))) +def test_token_type_cost_breakdown_bills_cache_tokens_at_the_provider_speed_multiplier( + monkeypatch: pytest.MonkeyPatch, speed: str | None, multiplier: float +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "fast-cache-model", + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 1e-7, + "cache_creation_input_token_cost": 1.25e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "provider_specific_entry": {"fast": 6.0}, + }, + ) + usage = Usage( + prompt_tokens=800, + completion_tokens=50, + total_tokens=850, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=400, cache_creation_tokens=300), + speed=speed, + ) + + breakdown = get_token_type_cost_breakdown(model="fast-cache-model", custom_llm_provider="anthropic", usage=usage) + + assert breakdown.cache_read_cost == pytest.approx(multiplier * 400 * 1e-7) + assert breakdown.cache_creation_cost == pytest.approx(multiplier * 300 * 1.25e-6)