From e52ae2baaf15ce4092e17191af1b5bdcaed99f85 Mon Sep 17 00:00:00 2001 From: liming Date: Mon, 7 Sep 2026 20:38:56 +0800 Subject: [PATCH] fix(dashscope): use response cache mode for pricing --- litellm/cost_calculator.py | 52 +------ litellm/llms/dashscope/cost_calculator.py | 13 +- litellm/types/utils.py | 1 - .../test_dashscope_cost_calculator.py | 129 +++++++----------- 4 files changed, 51 insertions(+), 144 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 67fc1a2b604..9a9d2ceda03 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -93,7 +93,6 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.llms.openai import ( - AllMessageValues, HttpxBinaryResponseContent, ImageGenerationRequestQuality, OpenAIModerationResponse, @@ -129,7 +128,6 @@ from litellm.utils import ( TextCompletionResponse, TranscriptionResponse, _cached_get_model_info_helper, - is_cached_message, token_counter, ) @@ -190,28 +188,6 @@ _SEARCH_CALL_TYPES: Final = frozenset( _AREALTIME_CALL_TYPE: Final = CallTypes.arealtime.value _MCP_CALL_TYPE: Final = CallTypes.call_mcp_tool.value -_DASHSCOPE_PROVIDERS: Final = frozenset(("dashscope", "qwencloud", "qwen_ai_platform")) - - -def _get_dashscope_cache_read_mode( - model: str, - custom_llm_provider: str | None, - usage: Usage, - cached_tokens: int, - cache_control_requested: bool, -) -> Literal["explicit", "implicit"] | None: - if custom_llm_provider not in _DASHSCOPE_PROVIDERS and not any( - model.startswith(f"{provider}/") for provider in _DASHSCOPE_PROVIDERS - ): - return None - - from litellm.llms.dashscope.cost_calculator import get_cache_read_mode - - return get_cache_read_mode( - usage=usage, - cached_tokens=cached_tokens, - cache_control_requested=cache_control_requested, - ) def _cost_per_token_custom_pricing_helper( @@ -220,7 +196,6 @@ def _cost_per_token_custom_pricing_helper( response_time_ms: float | None = 0.0, cached_tokens: float = 0, cache_creation_tokens: float = 0, - cache_read_mode: Literal["explicit", "implicit"] | None = None, ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, @@ -239,8 +214,8 @@ def _cost_per_token_custom_pricing_helper( output_cost_per_token: Final = custom_cost_per_token["output_cost_per_token"] cache_read_input_token_cost: Final = custom_cost_per_token.get( - "implicit_cache_read_input_token_cost" if cache_read_mode == "implicit" else "cache_read_input_token_cost", - custom_cost_per_token.get("cache_read_input_token_cost", input_cost_per_token), + "cache_read_input_token_cost", + input_cost_per_token, ) cache_creation_input_token_cost: Final = custom_cost_per_token.get( "cache_creation_input_token_cost", @@ -365,7 +340,6 @@ def cost_per_token( response: Any | None = None, ### REQUEST MODEL ### request_model: str | None = None, # original request model for router detection - cache_control_requested: bool = False, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -444,21 +418,12 @@ def cost_per_token( if _is_anthropic_style: _normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens - _cache_read_mode: Final = _get_dashscope_cache_read_mode( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage_block, - cached_tokens=int(_cache_read_tokens), - cache_control_requested=cache_control_requested, - ) - response_cost: Final = _cost_per_token_custom_pricing_helper( prompt_tokens=_normalized_prompt_tokens, completion_tokens=completion_tokens, response_time_ms=response_time_ms, cached_tokens=_cache_read_tokens, cache_creation_tokens=_cache_creation_tokens, - cache_read_mode=_cache_read_mode, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, ) @@ -690,12 +655,7 @@ def cost_per_token( cost_per_token as dashscope_cost_per_token, ) - return dashscope_cost_per_token( - model=model, - usage=usage_block, - custom_llm_provider=custom_llm_provider, - cache_control_requested=cache_control_requested, - ) + return dashscope_cost_per_token(model=model, usage=usage_block, custom_llm_provider=custom_llm_provider) elif custom_llm_provider == "azure_ai": return azure_ai_cost_per_token( model=model, @@ -1223,7 +1183,7 @@ def completion_cost( completion_response: object | None = None, model: str | None = None, prompt="", - messages: Sequence[AllMessageValues] = (), + messages: list = [], completion="", total_time: float | None = 0.0, # used for replicate, sagemaker call_type: CallTypesLiteral | None = None, @@ -1700,10 +1660,6 @@ def completion_cost( vertex_location=vertex_location, response=completion_response, request_model=request_model_for_cost, - cache_control_requested=( - custom_llm_provider in _DASHSCOPE_PROVIDERS - and any(is_cached_message(message) for message in messages) - ), ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 90469cec56f..35a0fdc6edf 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -45,7 +45,6 @@ class TokenBreakdown: def get_cache_read_mode( usage: Usage, cached_tokens: int, - cache_control_requested: bool = False, ) -> QwenContextCacheMode | None: if cached_tokens <= 0: return None @@ -58,14 +57,11 @@ def get_cache_read_mode( return "explicit" if cache_type is not None: return None - if cache_control_requested: - return "explicit" return "implicit" def _extract_token_breakdown( usage: Usage, - cache_control_requested: bool = False, ) -> TokenBreakdown: prompt_details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = prompt_details["cache_hit_tokens"] @@ -84,7 +80,6 @@ def _extract_token_breakdown( cache_read_mode=get_cache_read_mode( usage=usage, cached_tokens=cached_tokens, - cache_control_requested=cache_control_requested, ), ) @@ -174,12 +169,8 @@ def get_token_breakdown_and_rates( model_info: ModelInfo, usage: Usage, current_time: datetime | None = None, - cache_control_requested: bool = False, ) -> tuple[TokenBreakdown, TokenRates]: - breakdown: Final = _extract_token_breakdown( - usage, - cache_control_requested=cache_control_requested, - ) + breakdown: Final = _extract_token_breakdown(usage) raw_tiers: Final = model_info.get("tiered_pricing") tiered_pricing: Final = ( cast( # cast-ok: ModelInfo's legacy tier type uses Any; the list check preserves the existing runtime boundary @@ -218,7 +209,6 @@ def cost_per_token( usage: Usage, custom_llm_provider: str = "dashscope", current_time: datetime | None = None, - cache_control_requested: bool = False, ) -> tuple[float, float]: """ Calculate cost per token for Dashscope models. @@ -240,7 +230,6 @@ def cost_per_token( model_info=model_info, usage=usage, current_time=current_time, - cache_control_requested=cache_control_requested, ) return _bill(breakdown, rates) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 57dac099c84..f712d0fe21d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -124,7 +124,6 @@ class CostPerToken(TypedDict, total=False): input_cost_per_token: Required[float] output_cost_per_token: Required[float] cache_read_input_token_cost: float - implicit_cache_read_input_token_cost: ReadOnly[float] cache_creation_input_token_cost: float diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index 5ba7c931df0..0cb3b8ce9bc 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -16,7 +16,6 @@ import pytest # Add the project root to Python path import litellm -from litellm.cost_calculator import cost_per_token as litellm_cost_per_token from litellm.litellm_core_utils.llm_cost_calc.utils import get_token_type_cost_breakdown from litellm.llms.dashscope.cost_calculator import ( cost_per_token as dashscope_cost_per_token, @@ -112,8 +111,19 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) - def test_dashscope_request_cache_control_is_explicit_mode_fallback(self): - litellm.model_cost["dashscope/qwen-cache-control-fallback-test"] = { + @pytest.mark.parametrize( + ("prompt_tokens_details", "expected_cache_rate"), + [ + ({"cached_tokens": 600, "cache_type": "ephemeral"}, 1e-07), + ({"cached_tokens": 600}, 2e-07), + ], + ) + def test_completion_cost_routes_dashscope_cache_mode_pricing( + self, + prompt_tokens_details: dict[str, int | str], + expected_cache_rate: float, + ): + litellm.model_cost["dashscope/qwen-completion-cache-mode-test"] = { "litellm_provider": "dashscope", "mode": "chat", "input_cost_per_token": 1e-06, @@ -121,19 +131,27 @@ class TestDashscopeCostCalculator: "cache_read_input_token_cost": 1e-07, "implicit_cache_read_input_token_cost": 2e-07, } - usage = Usage( - prompt_tokens=1000, - completion_tokens=0, - prompt_tokens_details={"cached_tokens": 600}, + response = litellm.ModelResponse( + model="qwen-completion-cache-mode-test", + usage=Usage( + prompt_tokens=1000, + completion_tokens=100, + total_tokens=1100, + prompt_tokens_details=prompt_tokens_details, + ), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-cache-control-fallback-test", - usage=usage, - cache_control_requested=True, + cost = litellm.completion_cost( + completion_response=response, + model="qwen-completion-cache-mode-test", + custom_llm_provider="dashscope", ) - assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) + assert math.isclose( + cost, + (400 * 1e-06) + (600 * expected_cache_rate) + (100 * 4e-06), + rel_tol=1e-10, + ) def test_dashscope_tiered_pricing_uses_mode_specific_cache_read_rate(self): self._register_tiered_model( @@ -158,7 +176,15 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) - def test_dashscope_cost_breakdown_uses_mode_specific_cache_read_rate(self): + @pytest.mark.parametrize( + ("cache_type", "expected_cache_rate"), + [("ephemeral", 1e-07), (None, 2e-07)], + ) + def test_dashscope_cost_breakdown_matches_mode_specific_total( + self, + cache_type: str | None, + expected_cache_rate: float, + ): litellm.model_cost["dashscope/qwen-cache-breakdown-mode-test"] = { "litellm_provider": "dashscope", "mode": "chat", @@ -167,87 +193,24 @@ class TestDashscopeCostCalculator: "cache_read_input_token_cost": 1e-07, "implicit_cache_read_input_token_cost": 2e-07, } + prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600} + if cache_type is not None: + prompt_tokens_details["cache_type"] = cache_type usage = Usage( prompt_tokens=1000, completion_tokens=0, - prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + prompt_tokens_details=prompt_tokens_details, ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-breakdown-mode-test", usage=usage) breakdown = get_token_type_cost_breakdown( model="qwen-cache-breakdown-mode-test", custom_llm_provider="dashscope", usage=usage, ) - assert math.isclose(breakdown.cache_read_cost, 600 * 1e-07, rel_tol=1e-10) - - def test_dashscope_custom_pricing_uses_mode_specific_cache_read_rate(self): - usage = Usage( - prompt_tokens=1000, - completion_tokens=0, - prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, - ) - - prompt_cost, _ = litellm_cost_per_token( - model="custom-qwen", - custom_llm_provider="dashscope", - usage_object=usage, - prompt_tokens=1000, - custom_cost_per_token={ - "input_cost_per_token": 1e-06, - "output_cost_per_token": 4e-06, - "implicit_cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost": 1e-07, - }, - ) - - assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) - - def test_completion_cost_detects_cache_control_request_fallback(self): - usage = Usage( - prompt_tokens=1000, - completion_tokens=100, - total_tokens=1100, - prompt_tokens_details={"cached_tokens": 600}, - ) - response = litellm.ModelResponse( - id="test-id", - created=1234567890, - model="custom-qwen", - object="chat.completion", - choices=[], - usage=usage, - ) - - cost = litellm.completion_cost( - completion_response=response, - model="custom-qwen", - custom_llm_provider="dashscope", - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "stable prefix", - "cache_control": {"type": "ephemeral"}, - } - ], - } - ], - custom_cost_per_token={ - "input_cost_per_token": 1e-06, - "output_cost_per_token": 4e-06, - "cache_read_input_token_cost": 1e-07, - "implicit_cache_read_input_token_cost": 2e-07, - }, - ) - - assert math.isclose( - cost, - (400 * 1e-06) + (600 * 1e-07) + (100 * 4e-06), - rel_tol=1e-10, - ) + assert math.isclose(breakdown.cache_read_cost, 600 * expected_cache_rate, rel_tol=1e-10) + assert math.isclose(prompt_cost, (400 * 1e-06) + breakdown.cache_read_cost, rel_tol=1e-10) def test_dashscope_tiered_pricing_within_first_tier(self): """