mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(dashscope): avoid cache pricing import cycle
This commit is contained in:
parent
4faa0359cb
commit
e68b357656
5 changed files with 186 additions and 46 deletions
|
|
@ -437,6 +437,7 @@ def _resolve_cache_read_cost_rate(
|
|||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
current_time: datetime | None,
|
||||
service_tier: str | None = None,
|
||||
) -> float | None:
|
||||
if custom_llm_provider is None:
|
||||
return None
|
||||
|
|
@ -450,10 +451,44 @@ def _resolve_cache_read_cost_rate(
|
|||
)
|
||||
if provider_config is None:
|
||||
return None
|
||||
return provider_config.get_cache_read_input_token_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
cache_read_cost_key: Final = provider_config.get_cache_read_input_token_cost_key(usage=usage)
|
||||
if cache_read_cost_key is None:
|
||||
return None
|
||||
|
||||
tier: Final = _select_priced_tier(model_info=model_info, usage=usage)
|
||||
if tier is not None and cache_read_cost_key in tier:
|
||||
cache_read_cost_rate = tier_rate(tier, cache_read_cost_key)
|
||||
else:
|
||||
model_cache_read_cost_rate: Final = _get_cost_per_unit(
|
||||
model_info,
|
||||
_get_service_tier_cost_key(cache_read_cost_key, service_tier),
|
||||
default_value=None,
|
||||
)
|
||||
if model_cache_read_cost_rate is not None:
|
||||
cache_read_cost_rate = model_cache_read_cost_rate
|
||||
elif tier is not None:
|
||||
cache_read_cost_rate = tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token")
|
||||
else:
|
||||
cache_read_cost_rate = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
_get_service_tier_cost_key("cache_read_input_token_cost", service_tier),
|
||||
default_value=_get_cost_per_unit(model_info, "input_cost_per_token") or 0.0,
|
||||
),
|
||||
)
|
||||
|
||||
off_peak: Final = _open_off_peak_block(model_info, current_time)
|
||||
if off_peak is None:
|
||||
return cache_read_cost_rate
|
||||
mode_specific_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get(cache_read_cost_key))
|
||||
generic_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get("cache_read_input_token_cost"))
|
||||
return (
|
||||
mode_specific_off_peak_rate
|
||||
if mode_specific_off_peak_rate is not None
|
||||
else generic_off_peak_rate
|
||||
if generic_off_peak_rate is not None
|
||||
else cache_read_cost_rate
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1451,6 +1486,7 @@ def _cost_map_billed_rates(
|
|||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=billing_time,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
if provider_cache_read_cost_rate is not None:
|
||||
cache_read_cost_rate = provider_cache_read_cost_rate
|
||||
|
|
@ -1591,6 +1627,7 @@ def calculate_prompt_caching_savings(
|
|||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=billed_at,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
if provider_cache_read_cost is not None:
|
||||
cache_read_cost = provider_cache_read_cost
|
||||
|
|
|
|||
|
|
@ -21,12 +21,10 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import datetime
|
||||
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.types.utils import ModelInfo, ModelResponse, Usage
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ..base_utils import (
|
||||
map_developer_role_to_system_role,
|
||||
|
|
@ -72,13 +70,11 @@ class BaseConfig(ABC):
|
|||
def __init__(self):
|
||||
pass
|
||||
|
||||
def get_cache_read_input_token_cost(
|
||||
def get_cache_read_input_token_cost_key(
|
||||
self,
|
||||
model_info: "ModelInfo",
|
||||
usage: "Usage",
|
||||
current_time: "datetime | None" = None,
|
||||
) -> float | None:
|
||||
"""Return a provider-specific cache-read rate, or None for shared pricing."""
|
||||
usage: object,
|
||||
) -> str | None:
|
||||
"""Return a provider-specific cache-read pricing key, or None for shared pricing."""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -3,35 +3,29 @@ Translates from OpenAI's `/v1/chat/completions` to DashScope's `/v1/chat/complet
|
|||
"""
|
||||
|
||||
from collections.abc import Coroutine
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, overload
|
||||
from typing import Any, Final, Literal, overload
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
|
||||
|
||||
class DashScopeChatConfig(OpenAIGPTConfig):
|
||||
def get_cache_read_input_token_cost(
|
||||
def get_cache_read_input_token_cost_key(
|
||||
self,
|
||||
model_info: "ModelInfo",
|
||||
usage: "Usage",
|
||||
current_time: "datetime | None" = None,
|
||||
) -> float | None:
|
||||
from litellm.llms.dashscope.cost_calculator import (
|
||||
resolve_cache_read_input_token_cost,
|
||||
)
|
||||
|
||||
return resolve_cache_read_input_token_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
)
|
||||
usage: object,
|
||||
) -> str | None:
|
||||
prompt_tokens_details: Final = getattr(usage, "prompt_tokens_details", None)
|
||||
if prompt_tokens_details is None:
|
||||
return None
|
||||
cached_tokens: Final = getattr(prompt_tokens_details, "cached_tokens", 0)
|
||||
cache_type: Final = getattr(prompt_tokens_details, "cache_type", None)
|
||||
if not isinstance(cached_tokens, (int, float)) or cached_tokens <= 0:
|
||||
return None
|
||||
if cache_type is None:
|
||||
return "implicit_cache_read_input_token_cost"
|
||||
return None
|
||||
|
||||
def remove_cache_control_flag_from_messages_and_tools(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -218,19 +218,6 @@ def get_token_breakdown_and_rates(
|
|||
return breakdown, rates
|
||||
|
||||
|
||||
def resolve_cache_read_input_token_cost(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
current_time: datetime | None,
|
||||
) -> float:
|
||||
_, rates = get_token_breakdown_and_rates(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
)
|
||||
return rates.cache_read_rate
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
model: str,
|
||||
usage: Usage,
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
calculate_prompt_caching_savings,
|
||||
get_token_type_cost_breakdown,
|
||||
)
|
||||
from litellm.llms.dashscope.chat.transformation import DashScopeChatConfig
|
||||
from litellm.llms.dashscope.cost_calculator import (
|
||||
cost_per_token as dashscope_cost_per_token,
|
||||
)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
ModelInfo,
|
||||
|
|
@ -69,6 +71,130 @@ class TestDashscopeCostCalculator:
|
|||
is None
|
||||
)
|
||||
|
||||
def test_provider_default_cache_rate_key_uses_shared_pricing(self):
|
||||
config = OpenAIGPTConfig()
|
||||
|
||||
assert config.get_cache_read_input_token_cost_key(usage=object()) is None
|
||||
assert isinstance(config.get_config(), dict)
|
||||
|
||||
def test_dashscope_cache_rate_key_requires_cached_tokens(self):
|
||||
config = DashScopeChatConfig()
|
||||
|
||||
assert config.get_cache_read_input_token_cost_key(usage=object()) is None
|
||||
assert (
|
||||
config.get_cache_read_input_token_cost_key(
|
||||
usage=Usage(
|
||||
prompt_tokens=1,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details={"cached_tokens": 0},
|
||||
)
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "current_time", "expected_cache_rate"),
|
||||
[
|
||||
(
|
||||
{
|
||||
"key": "qwen-tier-mode-rate-test",
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 2000],
|
||||
"input_cost_per_token": 1e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
}
|
||||
],
|
||||
},
|
||||
None,
|
||||
2e-07,
|
||||
),
|
||||
(
|
||||
{
|
||||
"key": "qwen-tier-generic-fallback-test",
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 2000],
|
||||
"input_cost_per_token": 1e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
}
|
||||
],
|
||||
},
|
||||
None,
|
||||
1e-07,
|
||||
),
|
||||
(
|
||||
{
|
||||
"key": "qwen-flat-generic-fallback-test",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
},
|
||||
None,
|
||||
1e-07,
|
||||
),
|
||||
(
|
||||
{
|
||||
"key": "qwen-off-peak-mode-rate-test",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "00:00-00:00",
|
||||
"implicit_cache_read_input_token_cost": 8e-08,
|
||||
},
|
||||
},
|
||||
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
|
||||
8e-08,
|
||||
),
|
||||
(
|
||||
{
|
||||
"key": "qwen-off-peak-generic-rate-test",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "00:00-00:00",
|
||||
"implicit_cache_read_input_token_cost": "invalid",
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
},
|
||||
},
|
||||
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
|
||||
5e-08,
|
||||
),
|
||||
(
|
||||
{
|
||||
"key": "qwen-off-peak-standard-fallback-test",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "00:00-00:00",
|
||||
},
|
||||
},
|
||||
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
|
||||
2e-07,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks(
|
||||
self,
|
||||
model_info: ModelInfo,
|
||||
current_time: datetime | None,
|
||||
expected_cache_rate: float,
|
||||
):
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details={"cached_tokens": 600},
|
||||
)
|
||||
|
||||
cache_rate = _resolve_cache_read_cost_rate(
|
||||
custom_llm_provider="dashscope",
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
)
|
||||
|
||||
assert cache_rate == expected_cache_rate
|
||||
|
||||
def test_dashscope_flat_pricing_fallback(self):
|
||||
"""
|
||||
Tests that the dashscope calculator falls back to flat pricing for models
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue