scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-25 05:54:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 126

feat(dashscope): add separate explicit and implicit cache pricing (#45567)

Browse source 
* feat(dashscope): price Qwen implicit context-cache reads separately

* fix(dashscope): normalize implicit cache price strings

* fix(dashscope): price explicit cache hits through an explicit_ override key

Keep cache_read_input_token_cost as the default (implicit) Qwen cache-read
rate, which the cost map already holds, and add
explicit_cache_read_input_token_cost for hits DashScope tags with
cache_type "ephemeral". Untyped and unknown cache types keep the base rate,
so older releases reading the remote cost map keep billing unchanged.

Add explicit cache-read and cache-creation rates to the qwen3.7-max rows.

* fix(types): accept explicit_cache_read_input_token_cost as a constructor keyword

* fix(dashscope): inherit the explicit cache price on custom deployments and keep tier rates all-or-nothing

Deployments that set only input and output prices now inherit
explicit_cache_read_input_token_cost from the cost map, so explicit hits bill
at the explicit rate. A pricing tier without its own explicit rate bills its
own cache_read rate instead of the model-level explicit one, matching how an
off-peak block falls back.

The DashScope calculator now delegates the explicit rate to the shared
resolver, and the register_model warning is back to main's wording.

* fix(dashscope): keep a deployment's own cache-read price for explicit hits and ignore a non-string cache_type

* fix(dashscope): keep cache_type through Responses usage and let a registered cache-read price cover explicit hits

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
limdeng 2026-10-20 12:54:35 +08:00 • committed by GitHub
parent da80552897
commit 3b2cc34c09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 872 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -356,7 +356,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
)
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None:
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None:
tiered_pricing: Final = model_info.get("tiered_pricing")
if not isinstance(tiered_pricing, list) or not tiered_pricing:
return None
@ -545,6 +545,71 @@ class TokenRates:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def resolve_explicit_cache_read_cost_rate(
model_info: ModelInfo,
usage: Usage,
current_time: datetime | None,
service_tier: str | None = None,
) -> float | None:
cache_read_cost_key: Final = "explicit_cache_read_input_token_cost"
service_cache_read_cost_key: Final = _get_service_tier_cost_key(cache_read_cost_key, service_tier)
if (
model_info.get(service_cache_read_cost_key) is None
and model_info.get(cache_read_cost_key) is None
and model_info.get("tiered_pricing") is None
and model_info.get("off_peak_pricing") is None
):
return None
prompt_tokens_details: Final = usage.prompt_tokens_details
cache_type: Final = (
getattr(prompt_tokens_details, "cache_type", None) if prompt_tokens_details is not None else None
)
if parse_prompt_tokens_details(usage)["cache_hit_tokens"] <= 0 or cache_type != "ephemeral":
return None
tier: Final = _select_priced_tier(model_info=model_info, usage=usage)
off_peak: Final = _open_off_peak_block(model_info, current_time)
model_cache_read_cost_rate: Final = _get_cost_per_unit(
model_info,
service_cache_read_cost_key,
default_value=None,
)
tier_has_explicit_rate: Final = tier is not None and cache_read_cost_key in tier
off_peak_has_explicit_rate: Final = (
off_peak is not None and _parse_off_peak_rate(off_peak.get(cache_read_cost_key)) is not None
)
if model_cache_read_cost_rate is None and not tier_has_explicit_rate and not off_peak_has_explicit_rate:
return None
cache_read_cost_rate: Final = (
tier_rate(tier, cache_read_cost_key)
if tier is not None and tier_has_explicit_rate
else tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token")
if tier is not None
else model_cache_read_cost_rate
if model_cache_read_cost_rate is not None
else _get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_token_cost", service_tier),
default_value=_get_cost_per_unit(model_info, "input_cost_per_token") or 0.0,
)
or 0.0
)
if off_peak is None:
return cache_read_cost_rate
mode_specific_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get(cache_read_cost_key))
generic_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get("cache_read_input_token_cost"))
return (
mode_specific_off_peak_rate
if mode_specific_off_peak_rate is not None
else generic_off_peak_rate
if generic_off_peak_rate is not None
else cache_read_cost_rate
)
def _parse_off_peak_rate(value: object) -> float | None:
if isinstance(value, bool):
return None
@ -1458,12 +1523,19 @@ def generic_cost_per_token(
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
model_info=resolved_model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=resolved_model_info,
prompt_base_cost=prompt_base_cost,
cache_read_cost=cache_read_cost,
cache_read_cost=effective_cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
service_tier=service_tier,
@ -1658,6 +1730,16 @@ def _cost_map_billed_rates(
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
provider_cache_read_cost_rate: Final = resolve_explicit_cache_read_cost_rate(
model_info=model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
effective_cache_read_cost_rate: Final = (
cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
)
reasoning_rate: Final = _resolve_billed_reasoning_rate(
model_info=model_info,
usage=usage,
@ -1679,9 +1761,9 @@ def _cost_map_billed_rates(
return BilledTokenRates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
cache_read_input_token_cost=cache_read_cost_rate,
cache_read_input_token_cost=effective_cache_read_cost_rate,
cache_read_input_audio_token_cost=(
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
),
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
@ -1806,6 +1888,13 @@ def calculate_prompt_caching_savings(
current_time=billed_at,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
provider_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
model_info=model_info,
usage=usage,
current_time=billed_at,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost
write_rate: Final = cache_creation_cost or prompt_base_cost
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
@ -1820,7 +1909,7 @@ def calculate_prompt_caching_savings(
if details is not None
else None
)
read_discount: Final = cache_read_tokens * max(prompt_base_cost - cache_read_cost, 0.0)
read_discount: Final = cache_read_tokens * max(prompt_base_cost - effective_cache_read_cost, 0.0)
write_premium: Final = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_details,

13
litellm/llms/dashscope/cost_calculator.py
View file

@ -7,7 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
"""
from dataclasses import dataclass
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Final
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
apply_off_peak_pricing,
parse_completion_tokens_details,
parse_prompt_tokens_details,
resolve_explicit_cache_read_cost_rate,
)
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import get_model_info
@ -137,5 +138,13 @@ def cost_per_token(
)
standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier)
rates: Final = apply_off_peak_pricing(model_info, current_time, standard_rates)
explicit_cache_read_rate: Final = resolve_explicit_cache_read_cost_rate(
model_info=model_info,
usage=usage,
current_time=current_time,
)
billed_rates: Final = (
rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate)
)
return _bill(breakdown, rates)
return _bill(breakdown, billed_rates)

6
litellm/model_prices_and_context_window_backup.json
View file

@ -18018,7 +18018,9 @@
]
},
"dashscope/qwen3.7-max": {
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
@ -19285,7 +19287,9 @@
]
},
"qwencloud/qwen3.7-max": {
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "qwencloud",
"max_input_tokens": 991808,
@ -20512,7 +20516,9 @@
]
},
"qwen_ai_platform/qwen3.7-max": {
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "qwen_ai_platform",
"max_input_tokens": 991808,

5
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -3406,6 +3406,10 @@ class LiteLLMCompletionResponsesConfig:
if cache_write_tokens is not None
else MappingProxyType({})
)
cache_type: Final = getattr(prompt_details, "cache_type", None)
cache_type_extra: Final[Mapping[str, str]] = (
MappingProxyType({"cache_type": cache_type}) if cache_type is not None else MappingProxyType({})
)
# The cost path reads the grounding counters off the input details, and a realtime
# session's usage is rebuilt from its own response.done, so dropping them here bills
# no per-query grounding fee at all.
@ -3432,6 +3436,7 @@ class LiteLLMCompletionResponsesConfig:
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
**cache_type_extra,
**grounding_request_counts,
)

1
litellm/responses/utils.py
View file

@ -1259,6 +1259,7 @@ class ResponseAPILoggingUtils:
),
video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None),
web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None),
google_maps_grounding_requests=getattr(
response_api_usage.input_tokens_details, "google_maps_grounding_requests", None

20
litellm/router.py
View file

@ -9815,7 +9815,8 @@ class Router:
does not silently bill cache_read/cache_creation at 0.
User-specified cache fields always win; only ``None``/missing entries
are inherited. No-op when the backend model has no canonical entry.
are inherited, and a user-set cache_read price also covers explicit
cache hits. No-op when the backend model has no canonical entry.
"""
cache_fields: Final = (
"cache_creation_input_token_cost",
@ -9824,17 +9825,22 @@ class Router:
"cache_read_input_token_cost",
"cache_read_input_token_cost_above_200k_tokens",
)
if all(model_info.get(f) is not None for f in cache_fields):
inherited_fields: Final = (
cache_fields
if model_info.get("cache_read_input_token_cost") is not None
else (*cache_fields, "explicit_cache_read_input_token_cost")
)
missing_fields: Final = tuple(f for f in inherited_fields if model_info.get(f) is None)
if not missing_fields:
return
try:
backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider)
except Exception:
return
for field in cache_fields:
if model_info.get(field) is None:
backend_value = backend_info.get(field)
if backend_value is not None:
model_info[field] = backend_value
for field in missing_fields:
backend_value = backend_info.get(field)
if backend_value is not None:
model_info[field] = backend_value
@staticmethod
def _cost_map_backend_model(deployment: Deployment) -> str:

12
litellm/types/utils.py
View file

@ -291,6 +291,7 @@ class OffPeakPricing(TypedDict, total=False):
output_cost_per_token: ReadOnly[float]
output_cost_per_reasoning_token: ReadOnly[float]
cache_read_input_token_cost: ReadOnly[float]
explicit_cache_read_input_token_cost: ReadOnly[float]
cache_creation_input_token_cost: ReadOnly[float]
@ -320,6 +321,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_image_token_cost: ReadOnly[float | None]
explicit_cache_read_input_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_read_input_token_cost_balanced: ReadOnly[float | None]
@ -1840,6 +1842,13 @@ class PromptTokensDetailsWrapper(
cached_tokens_details: CachedTokensDetails | None = None
"""Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split."""
cache_type: str | None = None
"""Provider-reported cache mode. DashScope returns ``ephemeral`` for explicit Qwen Context Cache requests."""
@field_validator("cache_type", mode="before")
@classmethod
def _drop_non_string_cache_type(cls, value: object) -> str | None:
return value if isinstance(value, str) else None
def __setattr__(self, name: str, value: object) -> None:
super().__setattr__(name, value)
@ -1889,6 +1898,8 @@ class PromptTokensDetailsWrapper(
del self.cache_creation_token_details
if self.cached_tokens_details is None:
del self.cached_tokens_details
if self.cache_type is None:
del self.cache_type
class ServerToolUse(LiteLLMBaseModel):
@ -3921,6 +3932,7 @@ class MirroredPricingParams(LiteLLMBaseModel):
input_cost_per_character: float | None = None
output_cost_per_character: float | None = None
cache_read_input_token_cost: float | None = None
explicit_cache_read_input_token_cost: float | None = None
cache_creation_input_token_cost: float | None = None
cache_creation_input_token_cost_above_1hr: float | None = None
tiered_pricing: list[dict[str, Any]] | None = None

15
litellm/utils.py
View file

@ -3670,6 +3670,15 @@ def _registration_base(
return _RegistrationWrite(key=key, entry=dict(fallback.entry))
def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]:
return (
frozenset({"explicit_cache_read_input_token_cost"})
if value.get("cache_read_input_token_cost") is not None
and value.get("explicit_cache_read_input_token_cost") is None
else frozenset()
)
def _registration_write(
key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None
) -> _RegistrationWrite:
@ -3684,7 +3693,10 @@ def _registration_write(
for name, field_value in base.entry.items()
if name != "litellm_provider" or field_value is not None
}
return _RegistrationWrite(key=base.key, entry=update_dictionary(existing_model, value))
return _RegistrationWrite(
key=base.key,
entry={**update_dictionary(existing_model, value), **dict.fromkeys(_base_fields_replaced_by(value))},
)
def _registration_attempt(
@ -6614,6 +6626,7 @@ def get_model_info_helper(
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens", None

6
model_prices_and_context_window.json
View file

@ -18018,7 +18018,9 @@
]
},
"dashscope/qwen3.7-max": {
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
@ -19285,7 +19287,9 @@
]
},
"qwencloud/qwen3.7-max": {
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "qwencloud",
"max_input_tokens": 991808,
@ -20512,7 +20516,9 @@
]
},
"qwen_ai_platform/qwen3.7-max": {
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "qwen_ai_platform",
"max_input_tokens": 991808,

4
model_prices_and_context_window.schema.json
View file

@ -297,6 +297,10 @@
"format": "date",
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$"
},
"explicit_cache_read_input_token_cost": {
"type": "number",
"minimum": 0
},
"gemini_audio_only_live": {
"type": "boolean"
},

506
tests/unit/llms/dashscope/test_dashscope_cost_calculator.py
View file

@ -16,11 +16,18 @@ import pytest
# Add the project root to Python path
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import (
resolve_explicit_cache_read_cost_rate,
calculate_prompt_caching_savings,
generic_cost_per_token,
get_token_type_cost_breakdown,
)
from litellm.llms.dashscope.cost_calculator import (
cost_per_token as dashscope_cost_per_token,
)
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
ModelInfo,
PromptTokensDetailsWrapper,
Usage,
)
@ -35,6 +42,196 @@ class TestDashscopeCostCalculator:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
@pytest.mark.parametrize(
"prompt_tokens_details",
[
{"cached_tokens": 0, "cache_type": "ephemeral"},
{"cached_tokens": 600},
{"cached_tokens": 600, "cache_type": "unknown-future-type"},
],
)
def test_explicit_cache_rate_requires_an_ephemeral_cache_hit(
self,
prompt_tokens_details: dict[str, int | str],
):
assert (
resolve_explicit_cache_read_cost_rate(
model_info={
"key": "qwen-cache-rate-guard-test",
"explicit_cache_read_input_token_cost": 1e-07,
},
usage=Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details=prompt_tokens_details,
),
current_time=None,
)
is None
)
@pytest.mark.parametrize(
("model_info", "current_time", "expected_cache_rate"),
[
(
{
"key": "qwen-tier-mode-rate-test",
"tiered_pricing": [
{
"range": [0, 2000],
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
}
],
},
None,
1e-07,
),
(
{
"key": "qwen-tier-generic-fallback-test",
"tiered_pricing": [
{
"range": [0, 2000],
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 2e-07,
}
],
},
None,
None,
),
(
{
"key": "qwen-flat-generic-fallback-test",
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 2e-07,
},
None,
None,
),
(
{
"key": "qwen-invalid-explicit-rate-test",
"input_cost_per_token": 1e-06,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"explicit_cache_read_input_token_cost": True,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
None,
),
(
{
"key": "qwen-tier-off-peak-mode-rate-test",
"tiered_pricing": [
{
"range": [0, 2000],
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 2e-07,
}
],
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"explicit_cache_read_input_token_cost": 5e-08,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
5e-08,
),
(
{
"key": "qwen-flat-off-peak-mode-rate-test",
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 2e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"explicit_cache_read_input_token_cost": 5e-08,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
5e-08,
),
(
{
"key": "qwen-off-peak-mode-rate-test",
"input_cost_per_token": 1e-06,
"explicit_cache_read_input_token_cost": 1e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"explicit_cache_read_input_token_cost": 5e-08,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
5e-08,
),
(
{
"key": "qwen-off-peak-generic-rate-test",
"input_cost_per_token": 1e-06,
"explicit_cache_read_input_token_cost": 1e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"explicit_cache_read_input_token_cost": "invalid",
"cache_read_input_token_cost": 8e-08,
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
8e-08,
),
(
{
"key": "qwen-off-peak-standard-fallback-test",
"input_cost_per_token": 1e-06,
"explicit_cache_read_input_token_cost": 1e-07,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
1e-07,
),
],
)
def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks(
self,
model_info: ModelInfo,
current_time: datetime | None,
expected_cache_rate: float | None,
):
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
cache_rate = resolve_explicit_cache_read_cost_rate(
model_info=model_info,
usage=usage,
current_time=current_time,
)
assert cache_rate == expected_cache_rate
def test_provider_neutral_cache_rate_resolver_handles_service_tier_rate(self):
cache_rate = resolve_explicit_cache_read_cost_rate(
model_info={
"key": "qwen-service-tier-mode-rate-test",
"explicit_cache_read_input_token_cost_priority": 3e-07,
},
usage=Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
),
current_time=None,
service_tier="priority",
)
assert cache_rate == 3e-07
def test_dashscope_flat_pricing_fallback(self):
"""
Tests that the dashscope calculator falls back to flat pricing for models
@ -53,6 +250,276 @@ class TestDashscopeCostCalculator:
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
@pytest.mark.parametrize("explicit_cache_rate", [1e-07, "1e-7"])
@pytest.mark.parametrize(
("cache_type", "expected_cache_rate"),
[
("ephemeral", 1e-07),
(None, 2e-07),
("unknown-future-type", 2e-07),
],
)
def test_dashscope_flat_pricing_distinguishes_explicit_and_implicit_cache_reads(
self, cache_type: str | None, expected_cache_rate: float, explicit_cache_rate: float | str
) -> None:
litellm.model_cost["dashscope/qwen-context-cache-mode-test"] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"explicit_cache_read_input_token_cost": explicit_cache_rate,
"cache_read_input_token_cost": 2e-07,
}
prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600}
if cache_type is not None:
prompt_tokens_details["cache_type"] = cache_type
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details=prompt_tokens_details,
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-context-cache-mode-test", usage=usage)
assert math.isclose(
prompt_cost,
(400 * 1e-06) + (600 * expected_cache_rate),
rel_tol=1e-10,
)
def test_dashscope_explicit_cache_read_falls_back_to_existing_cache_read_rate(self):
litellm.model_cost["dashscope/qwen-explicit-cache-fallback-test"] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 1e-07,
}
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-explicit-cache-fallback-test",
usage=usage,
)
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
@pytest.mark.parametrize(
("prompt_tokens_details", "expected_cache_rate"),
[
({"cached_tokens": 600, "cache_type": "ephemeral"}, 1e-07),
({"cached_tokens": 600}, 2e-07),
],
)
def test_completion_cost_routes_dashscope_cache_mode_pricing(
self,
prompt_tokens_details: dict[str, int | str],
expected_cache_rate: float,
):
litellm.model_cost["dashscope/qwen-completion-cache-mode-test"] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
}
response = litellm.ModelResponse(
model="qwen-completion-cache-mode-test",
usage=Usage(
prompt_tokens=1000,
completion_tokens=100,
total_tokens=1100,
prompt_tokens_details=prompt_tokens_details,
),
)
cost = litellm.completion_cost(
completion_response=response,
model="qwen-completion-cache-mode-test",
custom_llm_provider="dashscope",
)
assert math.isclose(
cost,
(400 * 1e-06) + (600 * expected_cache_rate) + (100 * 4e-06),
rel_tol=1e-10,
)
def test_dashscope_tiered_pricing_uses_mode_specific_cache_read_rate(self):
self._register_tiered_model(
"dashscope/qwen-tiered-cache-mode-test",
[
{
"range": [0, 1000],
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"explicit_cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost": 2e-07,
}
],
)
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-tiered-cache-mode-test", usage=usage)
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
@pytest.mark.parametrize(
("explicit_cache_rate", "expected_cache_rate"),
[(1e-07, 1e-07), ("1e-7", 1e-07), (0.0, 0.0), ("0", 0.0)],
)
def test_dashscope_flat_explicit_cache_read_rate_wins_even_at_zero(
self, explicit_cache_rate: float | str, expected_cache_rate: float
) -> None:
litellm.model_cost["dashscope/qwen-flat-explicit-zero-rate-test"] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": explicit_cache_rate,
}
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-flat-explicit-zero-rate-test",
usage=usage,
)
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * expected_cache_rate), rel_tol=1e-10)
def test_dashscope_tier_without_an_explicit_rate_bills_that_tiers_cache_read_rate(self):
litellm.model_cost["dashscope/qwen-two-tier-explicit-fallback-test"] = {
"litellm_provider": "dashscope",
"mode": "chat",
"explicit_cache_read_input_token_cost": 4e-08,
"tiered_pricing": [
{
"range": [0, 1000],
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 8e-08,
},
{
"range": [1000, 10000],
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 4.8e-06,
"cache_read_input_token_cost": 2.4e-07,
},
],
}
usage = Usage(
prompt_tokens=3000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 2000, "cache_type": "ephemeral"},
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-two-tier-explicit-fallback-test", usage=usage)
breakdown = get_token_type_cost_breakdown(
model="qwen-two-tier-explicit-fallback-test",
custom_llm_provider="dashscope",
usage=usage,
)
assert math.isclose(breakdown.cache_read_cost, 2000 * 2.4e-07, rel_tol=1e-10)
assert math.isclose(prompt_cost, (1000 * 1.2e-06) + breakdown.cache_read_cost, rel_tol=1e-10)
def test_dashscope_prompt_caching_savings_uses_explicit_cache_read_rate(self):
model_info: ModelInfo = {
"key": "qwen-cache-savings-mode-test",
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
}
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
savings = calculate_prompt_caching_savings(
model_info=model_info,
usage=usage,
custom_llm_provider="dashscope",
)
assert math.isclose(savings, 600 * (1e-06 - 1e-07), rel_tol=1e-10)
def test_openai_compatible_qwen_total_uses_explicit_cache_read_rate(self):
model_info: ModelInfo = {
"key": "qwen-generic-cache-mode-test",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
}
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
prompt_cost, _ = generic_cost_per_token(
model="custom-qwen",
usage=usage,
custom_llm_provider="openai",
model_info=model_info,
)
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
@pytest.mark.parametrize(
("cache_type", "expected_cache_rate"),
[("ephemeral", 1e-07), (None, 2e-07)],
)
def test_dashscope_cost_breakdown_matches_mode_specific_total(
self,
cache_type: str | None,
expected_cache_rate: float,
):
litellm.model_cost["dashscope/qwen-cache-breakdown-mode-test"] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
}
prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600}
if cache_type is not None:
prompt_tokens_details["cache_type"] = cache_type
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details=prompt_tokens_details,
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-breakdown-mode-test", usage=usage)
breakdown = get_token_type_cost_breakdown(
model="qwen-cache-breakdown-mode-test",
custom_llm_provider="dashscope",
usage=usage,
)
assert math.isclose(breakdown.cache_read_cost, 600 * expected_cache_rate, rel_tol=1e-10)
assert math.isclose(prompt_cost, (400 * 1e-06) + breakdown.cache_read_cost, rel_tol=1e-10)
def test_dashscope_tiered_pricing_within_first_tier(self):
"""
Tests the dashscope tiered pricing when the request's input falls in the first tier.
@ -577,6 +1044,45 @@ class TestDashscopeCostCalculator:
assert math.isclose(peak_prompt_cost, (600 * 2.4e-06) + (300 * 2e-07) + (100 * 3e-06), rel_tol=1e-10)
assert math.isclose(peak_completion_cost, 200 * 4.8e-06, rel_tol=1e-10)
@pytest.mark.parametrize(
("cache_type", "expected_cache_rate"),
[("ephemeral", 5e-08), (None, 8e-08)],
)
def test_dashscope_off_peak_uses_mode_specific_cache_read_rate(
self, cache_type: str | None, expected_cache_rate: float
):
model_key = "dashscope/qwen-cache-mode-off-peak-test"
litellm.model_cost[model_key] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 2e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
"off_peak_pricing": {
"hours_utc": self.OFF_PEAK_WINDOW,
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 8e-08,
"explicit_cache_read_input_token_cost": 5e-08,
},
}
prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600}
if cache_type is not None:
prompt_tokens_details["cache_type"] = cache_type
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details=prompt_tokens_details,
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-cache-mode-off-peak-test",
usage=usage,
current_time=self.INSIDE_WINDOW,
)
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * expected_cache_rate), rel_tol=1e-10)
def test_dashscope_off_peak_window_overrides_the_selected_tier(self):
"""An open off-peak window bills the whole request at the flat off-peak rates, whichever tier
the input volume selected."""

1
tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -556,6 +556,7 @@ def test_get_logging_payload_maps_nested_cache_creation_input_tokens():
assert additional_usage_values["cache_creation_input_tokens"] == 2048
assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 2048
assert additional_usage_values["prompt_tokens_details"]["cache_type"] == "ephemeral"
def test_get_logging_payload_preserves_anthropic_cache_creation_input_tokens():

46
tests/unit/responses/test_responses_utils.py
View file

@ -7,9 +7,11 @@ import pytest
import litellm
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.utils import Usage
from litellm.utils import _invalidate_model_cost_lowercase_map
class TestResponsesAPIRequestUtils:
@ -890,3 +892,45 @@ class TestResponsesInputToChatMessages:
assert ResponsesAPIRequestUtils.responses_input_to_chat_messages(
[reasoning_item, user_message, "stray"]
) == [user_message]
def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate():
model = "dashscope/qwen-responses-explicit-cache-test"
previous_row = litellm.model_cost.get(model)
litellm.model_cost[model] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 2.5e-06,
"output_cost_per_token": 7.5e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
}
_invalidate_model_cost_lowercase_map()
try:
chat_usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 1000, "cache_type": "ephemeral"},
)
responses_object = ResponsesAPIResponse(
id="resp_explicit_cache",
created_at=0,
model="qwen-responses-explicit-cache-test",
object="response",
status="completed",
output=[],
parallel_tool_calls=False,
tool_choice="auto",
tools=[],
usage=LiteLLMCompletionResponsesConfig.transform_chat_completion_usage_to_responses_usage(chat_usage),
)
assert litellm.completion_cost(completion_response=responses_object, model=model) == pytest.approx(
1000 * 2.5e-07
)
finally:
if previous_row is None:
litellm.model_cost.pop(model, None)
else:
litellm.model_cost[model] = previous_row
_invalidate_model_cost_lowercase_map()

61
tests/unit/test_router_model_cost_isolation.py
View file

@ -953,6 +953,67 @@ def test_partial_pricing_does_not_overwrite_explicit_cache_fields():
_restore_model_cost_entries(model_keys)
@pytest.mark.parametrize(
("deployment_cache_prices", "billed_cache_read_rate"),
[({}, 2.5e-07), ({"cache_read_input_token_cost": 1e-07}, 1e-07)],
ids=["inherits-builtin-explicit-rate", "own-cache-read-price-wins"],
)
def test_partial_custom_pricing_bills_explicit_cache_hits(
deployment_cache_prices: dict[str, float], billed_cache_read_rate: float
):
backend_model = "dashscope/qwen-explicit-cache-inherit-backend"
deploy_id = "qwen-deploy-partial-pricing"
model_keys = {
deploy_id: litellm.model_cost.get(deploy_id),
backend_model: litellm.model_cost.get(backend_model),
}
litellm.model_cost[backend_model] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 2.5e-06,
"output_cost_per_token": 7.5e-06,
"cache_creation_input_token_cost": 3.125e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
}
_invalidate_model_cost_lowercase_map()
try:
Router(
model_list=[
{
"model_name": "qwen-custom",
"litellm_params": {
"model": backend_model,
"api_key": "fake-key",
"input_cost_per_token": 2e-06,
"output_cost_per_token": 6e-06,
**deployment_cache_prices,
},
"model_info": {"id": deploy_id},
}
],
)
response = litellm.ModelResponse(
model=backend_model,
usage=litellm.Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 1000, "cache_type": "ephemeral"},
),
)
cost = litellm.completion_cost(
completion_response=response,
model=backend_model,
custom_pricing=True,
router_model_id=deploy_id,
)
assert cost == pytest.approx(1000 * billed_cache_read_rate)
finally:
_restore_model_cost_entries(model_keys)
def test_inherit_builtin_cache_pricing_fills_only_missing_fields():
"""Direct unit test of the helper: missing cache fields are filled from the
backend model's built-in entry, while an explicitly set cache field and the

47
tests/unit/test_utils.py
View file

@ -866,6 +866,7 @@ def validate_model_cost_values(model_data, exceptions=None):
"cache_creation_input_token_cost",
"cache_creation_input_audio_token_cost",
"cache_read_input_token_cost",
"explicit_cache_read_input_token_cost",
"cache_read_input_audio_token_cost",
"cache_read_input_image_token_cost",
"input_dbu_cost_per_token",
@ -964,6 +965,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_token_cost_priority": {"type": "number"},
"cache_creation_input_token_cost_ultrafast": {"type": "number"},
"cache_read_input_token_cost": {"type": "number"},
"explicit_cache_read_input_token_cost": {"type": "number"},
"cache_read_input_token_cost_above_32k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_100k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_100k_tokens_batches": {"type": "number"},
@ -7777,6 +7779,51 @@ def test_register_model_without_a_provider_never_inherits_another_providers_entr
_invalidate_model_cost_lowercase_map()
@pytest.mark.parametrize(
"registered_prices, billed_cache_read_rate",
[
pytest.param({"input_cost_per_token": 2e-06}, 2.5e-07, id="inherits-builtin-explicit-rate"),
pytest.param({"cache_read_input_token_cost": 1e-07}, 1e-07, id="own-cache-read-price-wins"),
pytest.param(
{"cache_read_input_token_cost": 1e-07, "explicit_cache_read_input_token_cost": 5e-08},
5e-08,
id="own-explicit-price-wins",
),
],
)
def test_register_model_over_a_builtin_row_bills_explicit_cache_hits(
monkeypatch: pytest.MonkeyPatch, registered_prices: dict[str, float], billed_cache_read_rate: float
) -> None:
from litellm.utils import _invalidate_model_cost_lowercase_map
key: Final = "dashscope/qwen-register-explicit-cache-test"
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
litellm.model_cost[key] = {
"litellm_provider": "dashscope",
"mode": "chat",
"input_cost_per_token": 2.5e-06,
"output_cost_per_token": 7.5e-06,
"cache_read_input_token_cost": 5e-07,
"explicit_cache_read_input_token_cost": 2.5e-07,
}
_invalidate_model_cost_lowercase_map()
try:
litellm.register_model({key: registered_prices}, persist_across_reloads=False)
explicit_hit: Final = ModelResponse(
usage=Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 1000, "cache_type": "ephemeral"},
)
)
assert litellm.completion_cost(completion_response=explicit_hit, model=key) == pytest.approx(
1000 * billed_cache_read_rate
)
finally:
_invalidate_model_cost_lowercase_map()
@pytest.mark.parametrize(
"model, litellm_params, expects_native_config",
[

40
tests/unit/types/test_types_utils.py
View file

@ -209,6 +209,46 @@ def test_prompt_tokens_details_maps_nested_cache_creation_input_tokens():
assert not hasattr(non_int, "cache_write_tokens")
def test_prompt_tokens_details_preserves_qwen_cache_type():
from litellm.types.utils import PromptTokensDetailsWrapper
details = PromptTokensDetailsWrapper(
cached_tokens=1889,
cache_creation_input_tokens=0,
cache_type="ephemeral",
)
assert details.cache_type == "ephemeral"
assert details.model_dump()["cache_type"] == "ephemeral"
@pytest.mark.parametrize("cache_type", [1, {"mode": "explicit"}, ["ephemeral"]])
def test_usage_ignores_a_non_string_cache_type(cache_type: object):
from litellm.types.utils import Usage
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": cache_type},
)
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cached_tokens == 600
assert getattr(usage.prompt_tokens_details, "cache_type", None) is None
def test_custom_pricing_accepts_qwen_explicit_cache_rate():
from litellm.types.utils import CustomPricingLiteLLMParams
pricing = CustomPricingLiteLLMParams(
cache_read_input_token_cost=2e-07,
explicit_cache_read_input_token_cost=1e-07,
)
assert pricing.cache_read_input_token_cost == 2e-07
assert pricing.explicit_cache_read_input_token_cost == 1e-07
def test_usage_server_tool_use_dict_is_coerced_and_round_trips():
from litellm.types.utils import ServerToolUse, Usage

6
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -35162,6 +35162,8 @@ export interface components {
default_api_key_tpm_limit?: number | null;
/** Drop Params */
drop_params?: boolean | string | null;
/** Explicit Cache Read Input Token Cost */
explicit_cache_read_input_token_cost?: number | null;
/**
* Fireworks Forward User Id
* @description Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests.
@ -50058,6 +50060,8 @@ export interface components {
discoverable?: boolean | null;
/** Enable Tag Filtering */
enable_tag_filtering?: boolean | null;
/** Explicit Cache Read Input Token Cost */
explicit_cache_read_input_token_cost?: number | null;
/** Id */
id: string | null;
/** Input Cost Per Character */
@ -50412,6 +50416,8 @@ export interface components {
default_api_key_tpm_limit?: number | null;
/** Drop Params */
drop_params?: boolean | string | null;
/** Explicit Cache Read Input Token Cost */
explicit_cache_read_input_token_cost?: number | null;
/**
* Fireworks Forward User Id
* @description Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests.