From e68b357656212e7896fd39fbd183b987a764e5da Mon Sep 17 00:00:00 2001 From: liming Date: Mon, 14 Sep 2026 13:22:02 +0800 Subject: [PATCH] fix(dashscope): avoid cache pricing import cycle --- .../litellm_core_utils/llm_cost_calc/utils.py | 45 ++++++- litellm/llms/base_llm/chat/transformation.py | 14 +- litellm/llms/dashscope/chat/transformation.py | 34 ++--- litellm/llms/dashscope/cost_calculator.py | 13 -- .../test_dashscope_cost_calculator.py | 126 ++++++++++++++++++ 5 files changed, 186 insertions(+), 46 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 63b234f4bcb..022d28c5053 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -437,6 +437,7 @@ def _resolve_cache_read_cost_rate( model_info: ModelInfo, usage: Usage, current_time: datetime | None, + service_tier: str | None = None, ) -> float | None: if custom_llm_provider is None: return None @@ -450,10 +451,44 @@ def _resolve_cache_read_cost_rate( ) if provider_config is None: return None - return provider_config.get_cache_read_input_token_cost( - model_info=model_info, - usage=usage, - current_time=current_time, + cache_read_cost_key: Final = provider_config.get_cache_read_input_token_cost_key(usage=usage) + if cache_read_cost_key is None: + return None + + tier: Final = _select_priced_tier(model_info=model_info, usage=usage) + if tier is not None and cache_read_cost_key in tier: + cache_read_cost_rate = tier_rate(tier, cache_read_cost_key) + else: + model_cache_read_cost_rate: Final = _get_cost_per_unit( + model_info, + _get_service_tier_cost_key(cache_read_cost_key, service_tier), + default_value=None, + ) + if model_cache_read_cost_rate is not None: + cache_read_cost_rate = model_cache_read_cost_rate + elif tier is not None: + cache_read_cost_rate = tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token") + else: + cache_read_cost_rate = cast( + float, + _get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_token_cost", service_tier), + default_value=_get_cost_per_unit(model_info, "input_cost_per_token") or 0.0, + ), + ) + + off_peak: Final = _open_off_peak_block(model_info, current_time) + if off_peak is None: + return cache_read_cost_rate + mode_specific_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get(cache_read_cost_key)) + generic_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get("cache_read_input_token_cost")) + return ( + mode_specific_off_peak_rate + if mode_specific_off_peak_rate is not None + else generic_off_peak_rate + if generic_off_peak_rate is not None + else cache_read_cost_rate ) @@ -1451,6 +1486,7 @@ def _cost_map_billed_rates( model_info=model_info, usage=usage, current_time=billing_time, + service_tier=service_tier, ) if provider_cache_read_cost_rate is not None: cache_read_cost_rate = provider_cache_read_cost_rate @@ -1591,6 +1627,7 @@ def calculate_prompt_caching_savings( model_info=model_info, usage=usage, current_time=billed_at, + service_tier=service_tier, ) if provider_cache_read_cost is not None: cache_read_cost = provider_cache_read_cost diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index e9fd27d04ab..7957114ee27 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -21,12 +21,10 @@ from litellm.types.llms.openai import ( ) if TYPE_CHECKING: - from datetime import datetime - import tiktoken from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper - from litellm.types.utils import ModelInfo, ModelResponse, Usage + from litellm.types.utils import ModelResponse from ..base_utils import ( map_developer_role_to_system_role, @@ -72,13 +70,11 @@ class BaseConfig(ABC): def __init__(self): pass - def get_cache_read_input_token_cost( + def get_cache_read_input_token_cost_key( self, - model_info: "ModelInfo", - usage: "Usage", - current_time: "datetime | None" = None, - ) -> float | None: - """Return a provider-specific cache-read rate, or None for shared pricing.""" + usage: object, + ) -> str | None: + """Return a provider-specific cache-read pricing key, or None for shared pricing.""" return None @classmethod diff --git a/litellm/llms/dashscope/chat/transformation.py b/litellm/llms/dashscope/chat/transformation.py index 6ce7843d85a..70da8700610 100644 --- a/litellm/llms/dashscope/chat/transformation.py +++ b/litellm/llms/dashscope/chat/transformation.py @@ -3,35 +3,29 @@ Translates from OpenAI's `/v1/chat/completions` to DashScope's `/v1/chat/complet """ from collections.abc import Coroutine -from typing import TYPE_CHECKING, Any, Final, Literal, overload +from typing import Any, Final, Literal, overload from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from ...openai.chat.gpt_transformation import OpenAIGPTConfig -if TYPE_CHECKING: - from datetime import datetime - - from litellm.types.utils import ModelInfo, Usage - class DashScopeChatConfig(OpenAIGPTConfig): - def get_cache_read_input_token_cost( + def get_cache_read_input_token_cost_key( self, - model_info: "ModelInfo", - usage: "Usage", - current_time: "datetime | None" = None, - ) -> float | None: - from litellm.llms.dashscope.cost_calculator import ( - resolve_cache_read_input_token_cost, - ) - - return resolve_cache_read_input_token_cost( - model_info=model_info, - usage=usage, - current_time=current_time, - ) + usage: object, + ) -> str | None: + prompt_tokens_details: Final = getattr(usage, "prompt_tokens_details", None) + if prompt_tokens_details is None: + return None + cached_tokens: Final = getattr(prompt_tokens_details, "cached_tokens", 0) + cache_type: Final = getattr(prompt_tokens_details, "cache_type", None) + if not isinstance(cached_tokens, (int, float)) or cached_tokens <= 0: + return None + if cache_type is None: + return "implicit_cache_read_input_token_cost" + return None def remove_cache_control_flag_from_messages_and_tools( self, diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index fd1b8ac9809..13db822f52e 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -218,19 +218,6 @@ def get_token_breakdown_and_rates( return breakdown, rates -def resolve_cache_read_input_token_cost( - model_info: ModelInfo, - usage: Usage, - current_time: datetime | None, -) -> float: - _, rates = get_token_breakdown_and_rates( - model_info=model_info, - usage=usage, - current_time=current_time, - ) - return rates.cache_read_rate - - def cost_per_token( model: str, usage: Usage, diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index ab07a576b2d..830177b0936 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -21,9 +21,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, get_token_type_cost_breakdown, ) +from litellm.llms.dashscope.chat.transformation import DashScopeChatConfig from litellm.llms.dashscope.cost_calculator import ( cost_per_token as dashscope_cost_per_token, ) +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.utils import ( CompletionTokensDetailsWrapper, ModelInfo, @@ -69,6 +71,130 @@ class TestDashscopeCostCalculator: is None ) + def test_provider_default_cache_rate_key_uses_shared_pricing(self): + config = OpenAIGPTConfig() + + assert config.get_cache_read_input_token_cost_key(usage=object()) is None + assert isinstance(config.get_config(), dict) + + def test_dashscope_cache_rate_key_requires_cached_tokens(self): + config = DashScopeChatConfig() + + assert config.get_cache_read_input_token_cost_key(usage=object()) is None + assert ( + config.get_cache_read_input_token_cost_key( + usage=Usage( + prompt_tokens=1, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 0}, + ) + ) + is None + ) + + @pytest.mark.parametrize( + ("model_info", "current_time", "expected_cache_rate"), + [ + ( + { + "key": "qwen-tier-mode-rate-test", + "tiered_pricing": [ + { + "range": [0, 2000], + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1e-07, + "implicit_cache_read_input_token_cost": 2e-07, + } + ], + }, + None, + 2e-07, + ), + ( + { + "key": "qwen-tier-generic-fallback-test", + "tiered_pricing": [ + { + "range": [0, 2000], + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1e-07, + } + ], + }, + None, + 1e-07, + ), + ( + { + "key": "qwen-flat-generic-fallback-test", + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1e-07, + }, + None, + 1e-07, + ), + ( + { + "key": "qwen-off-peak-mode-rate-test", + "input_cost_per_token": 1e-06, + "implicit_cache_read_input_token_cost": 2e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "implicit_cache_read_input_token_cost": 8e-08, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 8e-08, + ), + ( + { + "key": "qwen-off-peak-generic-rate-test", + "input_cost_per_token": 1e-06, + "implicit_cache_read_input_token_cost": 2e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "implicit_cache_read_input_token_cost": "invalid", + "cache_read_input_token_cost": 5e-08, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 5e-08, + ), + ( + { + "key": "qwen-off-peak-standard-fallback-test", + "input_cost_per_token": 1e-06, + "implicit_cache_read_input_token_cost": 2e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 2e-07, + ), + ], + ) + def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks( + self, + model_info: ModelInfo, + current_time: datetime | None, + expected_cache_rate: float, + ): + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600}, + ) + + cache_rate = _resolve_cache_read_cost_rate( + custom_llm_provider="dashscope", + model_info=model_info, + usage=usage, + current_time=current_time, + ) + + assert cache_rate == expected_cache_rate + def test_dashscope_flat_pricing_fallback(self): """ Tests that the dashscope calculator falls back to flat pricing for models