mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(dashscope): bill 1h cache writes at the 1h rate (#52331)
* fix(dashscope): bill 1h cache writes at the 1h rate DashScope reports the 5m/1h cache-write split as a nested prompt_tokens_details.cache_creation object. That split is now read into cache_creation_token_details, and the DashScope calculator prices the 1h tokens at cache_creation_input_token_cost_above_1hr, falling back to the regular write rate when no 1h rate is set. * Bill a DashScope tier without a 1h rate at the tier write rate
This commit is contained in:
parent
f35ece0594
commit
3115254fc3
3 changed files with 150 additions and 2 deletions
|
|
@ -10,25 +10,32 @@ See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
TokenRates,
|
||||
apply_off_peak_pricing,
|
||||
calculate_cache_writing_cost,
|
||||
parse_completion_tokens_details,
|
||||
parse_prompt_tokens_details,
|
||||
resolve_explicit_cache_read_cost_rate,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.types.utils import CacheCreationTokenDetails, ModelInfo, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
_LONG_TTL_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"1hr": "cache_creation_input_token_cost_above_1hr"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenBreakdown:
|
||||
text_tokens: int
|
||||
cached_tokens: int
|
||||
cache_creation_tokens: int
|
||||
cache_creation_token_details: CacheCreationTokenDetails | None
|
||||
completion_tokens: int
|
||||
reasoning_tokens: int
|
||||
|
||||
|
|
@ -50,6 +57,7 @@ def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
|
|||
text_tokens=text_tokens,
|
||||
cached_tokens=cached_tokens,
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=prompt_details["cache_creation_token_details"],
|
||||
completion_tokens=completion_tokens,
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
)
|
||||
|
|
@ -61,6 +69,16 @@ def _flat_rate(value: float | None, fallback_value: float | None) -> float:
|
|||
return float(value)
|
||||
|
||||
|
||||
def _long_ttl_cache_write_rates(pricing: Mapping[str, object]) -> Mapping[str, float]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
ttl: tier_rate(pricing, key)
|
||||
for ttl, key in _LONG_TTL_CACHE_WRITE_RATE_KEYS.items()
|
||||
if pricing.get(key) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _flat_rates(model_info: ModelInfo) -> TokenRates:
|
||||
reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token")
|
||||
return TokenRates(
|
||||
|
|
@ -73,6 +91,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates:
|
|||
),
|
||||
output_rate=float(model_info.get("output_cost_per_token") or 0.0),
|
||||
reasoning_rate=None if reasoning_rate is None else float(reasoning_rate),
|
||||
cache_creation_rates_by_ttl=_long_ttl_cache_write_rates(model_info),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -95,6 +114,7 @@ def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates
|
|||
if tier_declares_output
|
||||
else flat_rates.reasoning_rate
|
||||
),
|
||||
cache_creation_rates_by_ttl=_long_ttl_cache_write_rates(tier),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -102,7 +122,12 @@ def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]:
|
|||
prompt_cost: Final = (
|
||||
(breakdown.text_tokens * rates.input_rate)
|
||||
+ (breakdown.cached_tokens * rates.cache_read_rate)
|
||||
+ (breakdown.cache_creation_tokens * rates.cache_creation_rate)
|
||||
+ calculate_cache_writing_cost(
|
||||
cache_creation_tokens=breakdown.cache_creation_tokens,
|
||||
cache_creation_token_details=breakdown.cache_creation_token_details,
|
||||
cache_creation_cost=rates.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl,
|
||||
)
|
||||
)
|
||||
completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + (
|
||||
breakdown.reasoning_tokens * rates.billed_reasoning_rate
|
||||
|
|
|
|||
|
|
@ -1907,6 +1907,15 @@ class CacheCreationTokenDetails(LiteLLMBaseModel):
|
|||
ephemeral_12h_input_tokens: int | None = None
|
||||
|
||||
|
||||
def _nested_cache_creation_token_details(nested: object) -> CacheCreationTokenDetails | None:
|
||||
if not isinstance(nested, Mapping):
|
||||
return None
|
||||
try:
|
||||
return CacheCreationTokenDetails.model_validate(nested)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
class PromptTokensDetailsWrapper(
|
||||
SafeAttributeModel, PromptTokensDetails
|
||||
): # extends with image generation fields (text_tokens, image_tokens)
|
||||
|
|
@ -1991,6 +2000,8 @@ class PromptTokensDetailsWrapper(
|
|||
)
|
||||
)
|
||||
)
|
||||
if self.cache_creation_token_details is None and extra_fields is not None:
|
||||
self.cache_creation_token_details = _nested_cache_creation_token_details(extra_fields.get("cache_creation"))
|
||||
if self.character_count is None:
|
||||
del self.character_count
|
||||
if self.image_count is None:
|
||||
|
|
|
|||
|
|
@ -834,6 +834,118 @@ class TestDashscopeCostCalculator:
|
|||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pricing", "expected_one_hour_rate"),
|
||||
[
|
||||
pytest.param(
|
||||
{
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"cache_creation_input_token_cost": 1.875e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 3e-06,
|
||||
},
|
||||
3e-06,
|
||||
id="flat_one_hour_rate",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"cache_creation_input_token_cost": 1.875e-06,
|
||||
},
|
||||
1.875e-06,
|
||||
id="no_one_hour_rate_bills_the_write_rate",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"cache_creation_input_token_cost_above_1hr": 9e-06,
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 256000],
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"cache_creation_input_token_cost": 1.875e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 3e-06,
|
||||
}
|
||||
],
|
||||
},
|
||||
3e-06,
|
||||
id="tier_one_hour_rate_wins_over_the_model_rate",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"cache_creation_input_token_cost_above_1hr": 3e-06,
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 256000],
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"cache_creation_input_token_cost": 1.875e-06,
|
||||
}
|
||||
],
|
||||
},
|
||||
1.875e-06,
|
||||
id="tier_without_a_one_hour_rate_bills_the_tier_write_rate",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_dashscope_one_hour_cache_writes_bill_at_the_one_hour_rate(
|
||||
self, pricing: dict[str, object], expected_one_hour_rate: float
|
||||
):
|
||||
"""
|
||||
Regression (#52261): DashScope reports the 5m/1h cache-write split as a nested
|
||||
prompt_tokens_details.cache_creation object, and the 1h tokens must bill at
|
||||
cache_creation_input_token_cost_above_1hr instead of the 5m write rate.
|
||||
"""
|
||||
litellm.model_cost["dashscope/qwen-one-hour-cache-write-test"] = {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
**pricing,
|
||||
}
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=10016,
|
||||
completion_tokens=2,
|
||||
total_tokens=10018,
|
||||
prompt_tokens_details={
|
||||
"cached_tokens": 0,
|
||||
"cache_type": "ephemeral",
|
||||
"cache_creation_input_tokens": 10000,
|
||||
"cache_write_tokens": 10000,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 2000, "ephemeral_1h_input_tokens": 8000},
|
||||
"text_tokens": 10016,
|
||||
},
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-one-hour-cache-write-test", usage=usage)
|
||||
|
||||
expected_write_cost = (2000 * 1.875e-06) + (8000 * expected_one_hour_rate)
|
||||
assert math.isclose(prompt_cost, (16 * 1.5e-06) + expected_write_cost, rel_tol=1e-10)
|
||||
|
||||
breakdown = get_token_type_cost_breakdown(
|
||||
model="qwen-one-hour-cache-write-test", custom_llm_provider="dashscope", usage=usage
|
||||
)
|
||||
assert math.isclose(breakdown.cache_creation_cost, expected_write_cost, rel_tol=1e-10)
|
||||
|
||||
@pytest.mark.parametrize("malformed_split", ["1h", {"ephemeral_1h_input_tokens": "many"}])
|
||||
def test_dashscope_malformed_cache_write_split_bills_the_write_rate(self, malformed_split: object):
|
||||
litellm.model_cost["dashscope/qwen-malformed-cache-split-test"] = {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"cache_creation_input_token_cost": 1.875e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 3e-06,
|
||||
}
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=10016,
|
||||
completion_tokens=2,
|
||||
total_tokens=10018,
|
||||
prompt_tokens_details={"cache_creation_input_tokens": 10000, "cache_creation": malformed_split},
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-malformed-cache-split-test", usage=usage)
|
||||
|
||||
assert math.isclose(prompt_cost, (16 * 1.5e-06) + (10000 * 1.875e-06), rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_tier_without_an_output_rate_bills_the_model_rate(self):
|
||||
"""
|
||||
Regression: a tier declaring only an input rate served every completion for free,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue