fix(dashscope): avoid cache pricing import cycle

This commit is contained in:
liming 2026-09-14 13:22:02 +08:00
parent 4faa0359cb
commit e68b357656
5 changed files with 186 additions and 46 deletions

View file

@ -437,6 +437,7 @@ def _resolve_cache_read_cost_rate(
model_info: ModelInfo,
usage: Usage,
current_time: datetime | None,
service_tier: str | None = None,
) -> float | None:
if custom_llm_provider is None:
return None
@ -450,10 +451,44 @@ def _resolve_cache_read_cost_rate(
)
if provider_config is None:
return None
return provider_config.get_cache_read_input_token_cost(
model_info=model_info,
usage=usage,
current_time=current_time,
cache_read_cost_key: Final = provider_config.get_cache_read_input_token_cost_key(usage=usage)
if cache_read_cost_key is None:
return None
tier: Final = _select_priced_tier(model_info=model_info, usage=usage)
if tier is not None and cache_read_cost_key in tier:
cache_read_cost_rate = tier_rate(tier, cache_read_cost_key)
else:
model_cache_read_cost_rate: Final = _get_cost_per_unit(
model_info,
_get_service_tier_cost_key(cache_read_cost_key, service_tier),
default_value=None,
)
if model_cache_read_cost_rate is not None:
cache_read_cost_rate = model_cache_read_cost_rate
elif tier is not None:
cache_read_cost_rate = tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token")
else:
cache_read_cost_rate = cast(
float,
_get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_token_cost", service_tier),
default_value=_get_cost_per_unit(model_info, "input_cost_per_token") or 0.0,
),
)
off_peak: Final = _open_off_peak_block(model_info, current_time)
if off_peak is None:
return cache_read_cost_rate
mode_specific_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get(cache_read_cost_key))
generic_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get("cache_read_input_token_cost"))
return (
mode_specific_off_peak_rate
if mode_specific_off_peak_rate is not None
else generic_off_peak_rate
if generic_off_peak_rate is not None
else cache_read_cost_rate
)
@ -1451,6 +1486,7 @@ def _cost_map_billed_rates(
model_info=model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
if provider_cache_read_cost_rate is not None:
cache_read_cost_rate = provider_cache_read_cost_rate
@ -1591,6 +1627,7 @@ def calculate_prompt_caching_savings(
model_info=model_info,
usage=usage,
current_time=billed_at,
service_tier=service_tier,
)
if provider_cache_read_cost is not None:
cache_read_cost = provider_cache_read_cost

View file

@ -21,12 +21,10 @@ from litellm.types.llms.openai import (
)
if TYPE_CHECKING:
from datetime import datetime
import tiktoken
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.utils import ModelInfo, ModelResponse, Usage
from litellm.types.utils import ModelResponse
from ..base_utils import (
map_developer_role_to_system_role,
@ -72,13 +70,11 @@ class BaseConfig(ABC):
def __init__(self):
pass
def get_cache_read_input_token_cost(
def get_cache_read_input_token_cost_key(
self,
model_info: "ModelInfo",
usage: "Usage",
current_time: "datetime | None" = None,
) -> float | None:
"""Return a provider-specific cache-read rate, or None for shared pricing."""
usage: object,
) -> str | None:
"""Return a provider-specific cache-read pricing key, or None for shared pricing."""
return None
@classmethod

View file

@ -3,35 +3,29 @@ Translates from OpenAI's `/v1/chat/completions` to DashScope's `/v1/chat/complet
"""
from collections.abc import Coroutine
from typing import TYPE_CHECKING, Any, Final, Literal, overload
from typing import Any, Final, Literal, overload
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from datetime import datetime
from litellm.types.utils import ModelInfo, Usage
class DashScopeChatConfig(OpenAIGPTConfig):
def get_cache_read_input_token_cost(
def get_cache_read_input_token_cost_key(
self,
model_info: "ModelInfo",
usage: "Usage",
current_time: "datetime | None" = None,
) -> float | None:
from litellm.llms.dashscope.cost_calculator import (
resolve_cache_read_input_token_cost,
)
return resolve_cache_read_input_token_cost(
model_info=model_info,
usage=usage,
current_time=current_time,
)
usage: object,
) -> str | None:
prompt_tokens_details: Final = getattr(usage, "prompt_tokens_details", None)
if prompt_tokens_details is None:
return None
cached_tokens: Final = getattr(prompt_tokens_details, "cached_tokens", 0)
cache_type: Final = getattr(prompt_tokens_details, "cache_type", None)
if not isinstance(cached_tokens, (int, float)) or cached_tokens <= 0:
return None
if cache_type is None:
return "implicit_cache_read_input_token_cost"
return None
def remove_cache_control_flag_from_messages_and_tools(
self,

View file

@ -218,19 +218,6 @@ def get_token_breakdown_and_rates(
return breakdown, rates
def resolve_cache_read_input_token_cost(
model_info: ModelInfo,
usage: Usage,
current_time: datetime | None,
) -> float:
_, rates = get_token_breakdown_and_rates(
model_info=model_info,
usage=usage,
current_time=current_time,
)
return rates.cache_read_rate
def cost_per_token(
model: str,
usage: Usage,

View file

@ -21,9 +21,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
calculate_prompt_caching_savings,
get_token_type_cost_breakdown,
)
from litellm.llms.dashscope.chat.transformation import DashScopeChatConfig
from litellm.llms.dashscope.cost_calculator import (
cost_per_token as dashscope_cost_per_token,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
ModelInfo,
@ -69,6 +71,130 @@ class TestDashscopeCostCalculator:
is None
)
def test_provider_default_cache_rate_key_uses_shared_pricing(self):
config = OpenAIGPTConfig()
assert config.get_cache_read_input_token_cost_key(usage=object()) is None
assert isinstance(config.get_config(), dict)
def test_dashscope_cache_rate_key_requires_cached_tokens(self):
config = DashScopeChatConfig()
assert config.get_cache_read_input_token_cost_key(usage=object()) is None
assert (
config.get_cache_read_input_token_cost_key(
usage=Usage(
prompt_tokens=1,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 0},
)
)
is None
)
@pytest.mark.parametrize(
("model_info", "current_time", "expected_cache_rate"),
[
(
{
"key": "qwen-tier-mode-rate-test",
"tiered_pricing": [
{
"range": [0, 2000],
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 1e-07,
"implicit_cache_read_input_token_cost": 2e-07,
}
],
},
None,
2e-07,
),
(
{
"key": "qwen-tier-generic-fallback-test",
"tiered_pricing": [
{
"range": [0, 2000],
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 1e-07,
}
],
},
None,
1e-07,
),
(
{
"key": "qwen-flat-generic-fallback-test",
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 1e-07,
},
None,
1e-07,
),
(
{
"key": "qwen-off-peak-mode-rate-test",
"input_cost_per_token": 1e-06,
"implicit_cache_read_input_token_cost": 2e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"implicit_cache_read_input_token_cost": 8e-08,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
8e-08,
),
(
{
"key": "qwen-off-peak-generic-rate-test",
"input_cost_per_token": 1e-06,
"implicit_cache_read_input_token_cost": 2e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"implicit_cache_read_input_token_cost": "invalid",
"cache_read_input_token_cost": 5e-08,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
5e-08,
),
(
{
"key": "qwen-off-peak-standard-fallback-test",
"input_cost_per_token": 1e-06,
"implicit_cache_read_input_token_cost": 2e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
2e-07,
),
],
)
def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks(
self,
model_info: ModelInfo,
current_time: datetime | None,
expected_cache_rate: float,
):
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600},
)
cache_rate = _resolve_cache_read_cost_rate(
custom_llm_provider="dashscope",
model_info=model_info,
usage=usage,
current_time=current_time,
)
assert cache_rate == expected_cache_rate
def test_dashscope_flat_pricing_fallback(self):
"""
Tests that the dashscope calculator falls back to flat pricing for models