mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(dashscope): use response cache mode for pricing
This commit is contained in:
liming
parent
07dc234380
commit
e52ae2baaf
4 changed files with 51 additions and 144 deletions
|
|
@ -93,7 +93,6 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke
|
|||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
HttpxBinaryResponseContent,
|
||||
ImageGenerationRequestQuality,
|
||||
OpenAIModerationResponse,
|
||||
|
|
@ -129,7 +128,6 @@ from litellm.utils import (
|
|||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
_cached_get_model_info_helper,
|
||||
is_cached_message,
|
||||
token_counter,
|
||||
)
|
||||
|
||||
|
|
@ -190,28 +188,6 @@ _SEARCH_CALL_TYPES: Final = frozenset(
|
|||
|
||||
_AREALTIME_CALL_TYPE: Final = CallTypes.arealtime.value
|
||||
_MCP_CALL_TYPE: Final = CallTypes.call_mcp_tool.value
|
||||
_DASHSCOPE_PROVIDERS: Final = frozenset(("dashscope", "qwencloud", "qwen_ai_platform"))
|
||||
|
||||
|
||||
def _get_dashscope_cache_read_mode(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
usage: Usage,
|
||||
cached_tokens: int,
|
||||
cache_control_requested: bool,
|
||||
) -> Literal["explicit", "implicit"] | None:
|
||||
if custom_llm_provider not in _DASHSCOPE_PROVIDERS and not any(
|
||||
model.startswith(f"{provider}/") for provider in _DASHSCOPE_PROVIDERS
|
||||
):
|
||||
return None
|
||||
|
||||
from litellm.llms.dashscope.cost_calculator import get_cache_read_mode
|
||||
|
||||
return get_cache_read_mode(
|
||||
usage=usage,
|
||||
cached_tokens=cached_tokens,
|
||||
cache_control_requested=cache_control_requested,
|
||||
)
|
||||
|
||||
|
||||
def _cost_per_token_custom_pricing_helper(
|
||||
|
|
@ -220,7 +196,6 @@ def _cost_per_token_custom_pricing_helper(
|
|||
response_time_ms: float | None = 0.0,
|
||||
cached_tokens: float = 0,
|
||||
cache_creation_tokens: float = 0,
|
||||
cache_read_mode: Literal["explicit", "implicit"] | None = None,
|
||||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
|
|
@ -239,8 +214,8 @@ def _cost_per_token_custom_pricing_helper(
|
|||
output_cost_per_token: Final = custom_cost_per_token["output_cost_per_token"]
|
||||
|
||||
cache_read_input_token_cost: Final = custom_cost_per_token.get(
|
||||
"implicit_cache_read_input_token_cost" if cache_read_mode == "implicit" else "cache_read_input_token_cost",
|
||||
custom_cost_per_token.get("cache_read_input_token_cost", input_cost_per_token),
|
||||
"cache_read_input_token_cost",
|
||||
input_cost_per_token,
|
||||
)
|
||||
cache_creation_input_token_cost: Final = custom_cost_per_token.get(
|
||||
"cache_creation_input_token_cost",
|
||||
|
|
@ -365,7 +340,6 @@ def cost_per_token(
|
|||
response: Any | None = None,
|
||||
### REQUEST MODEL ###
|
||||
request_model: str | None = None, # original request model for router detection
|
||||
cache_control_requested: bool = False,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
||||
|
|
@ -444,21 +418,12 @@ def cost_per_token(
|
|||
if _is_anthropic_style:
|
||||
_normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
|
||||
|
||||
_cache_read_mode: Final = _get_dashscope_cache_read_mode(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage_block,
|
||||
cached_tokens=int(_cache_read_tokens),
|
||||
cache_control_requested=cache_control_requested,
|
||||
)
|
||||
|
||||
response_cost: Final = _cost_per_token_custom_pricing_helper(
|
||||
prompt_tokens=_normalized_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time_ms=response_time_ms,
|
||||
cached_tokens=_cache_read_tokens,
|
||||
cache_creation_tokens=_cache_creation_tokens,
|
||||
cache_read_mode=_cache_read_mode,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
)
|
||||
|
|
@ -690,12 +655,7 @@ def cost_per_token(
|
|||
cost_per_token as dashscope_cost_per_token,
|
||||
)
|
||||
|
||||
return dashscope_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
cache_control_requested=cache_control_requested,
|
||||
)
|
||||
return dashscope_cost_per_token(model=model, usage=usage_block, custom_llm_provider=custom_llm_provider)
|
||||
elif custom_llm_provider == "azure_ai":
|
||||
return azure_ai_cost_per_token(
|
||||
model=model,
|
||||
|
|
@ -1223,7 +1183,7 @@ def completion_cost(
|
|||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
prompt="",
|
||||
messages: Sequence[AllMessageValues] = (),
|
||||
messages: list = [],
|
||||
completion="",
|
||||
total_time: float | None = 0.0, # used for replicate, sagemaker
|
||||
call_type: CallTypesLiteral | None = None,
|
||||
|
|
@ -1700,10 +1660,6 @@ def completion_cost(
|
|||
vertex_location=vertex_location,
|
||||
response=completion_response,
|
||||
request_model=request_model_for_cost,
|
||||
cache_control_requested=(
|
||||
custom_llm_provider in _DASHSCOPE_PROVIDERS
|
||||
and any(is_cached_message(message) for message in messages)
|
||||
),
|
||||
)
|
||||
|
||||
# Get additional costs from provider (e.g., routing fees, infrastructure costs)
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ class TokenBreakdown:
|
|||
def get_cache_read_mode(
|
||||
usage: Usage,
|
||||
cached_tokens: int,
|
||||
cache_control_requested: bool = False,
|
||||
) -> QwenContextCacheMode | None:
|
||||
if cached_tokens <= 0:
|
||||
return None
|
||||
|
|
@ -58,14 +57,11 @@ def get_cache_read_mode(
|
|||
return "explicit"
|
||||
if cache_type is not None:
|
||||
return None
|
||||
if cache_control_requested:
|
||||
return "explicit"
|
||||
return "implicit"
|
||||
|
||||
|
||||
def _extract_token_breakdown(
|
||||
usage: Usage,
|
||||
cache_control_requested: bool = False,
|
||||
) -> TokenBreakdown:
|
||||
prompt_details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = prompt_details["cache_hit_tokens"]
|
||||
|
|
@ -84,7 +80,6 @@ def _extract_token_breakdown(
|
|||
cache_read_mode=get_cache_read_mode(
|
||||
usage=usage,
|
||||
cached_tokens=cached_tokens,
|
||||
cache_control_requested=cache_control_requested,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -174,12 +169,8 @@ def get_token_breakdown_and_rates(
|
|||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
current_time: datetime | None = None,
|
||||
cache_control_requested: bool = False,
|
||||
) -> tuple[TokenBreakdown, TokenRates]:
|
||||
breakdown: Final = _extract_token_breakdown(
|
||||
usage,
|
||||
cache_control_requested=cache_control_requested,
|
||||
)
|
||||
breakdown: Final = _extract_token_breakdown(usage)
|
||||
raw_tiers: Final = model_info.get("tiered_pricing")
|
||||
tiered_pricing: Final = (
|
||||
cast( # cast-ok: ModelInfo's legacy tier type uses Any; the list check preserves the existing runtime boundary
|
||||
|
|
@ -218,7 +209,6 @@ def cost_per_token(
|
|||
usage: Usage,
|
||||
custom_llm_provider: str = "dashscope",
|
||||
current_time: datetime | None = None,
|
||||
cache_control_requested: bool = False,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Calculate cost per token for Dashscope models.
|
||||
|
|
@ -240,7 +230,6 @@ def cost_per_token(
|
|||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
cache_control_requested=cache_control_requested,
|
||||
)
|
||||
|
||||
return _bill(breakdown, rates)
|
||||
|
|
|
|||
|
|
@ -124,7 +124,6 @@ class CostPerToken(TypedDict, total=False):
|
|||
input_cost_per_token: Required[float]
|
||||
output_cost_per_token: Required[float]
|
||||
cache_read_input_token_cost: float
|
||||
implicit_cache_read_input_token_cost: ReadOnly[float]
|
||||
cache_creation_input_token_cost: float
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import pytest
|
|||
|
||||
# Add the project root to Python path
|
||||
import litellm
|
||||
from litellm.cost_calculator import cost_per_token as litellm_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_token_type_cost_breakdown
|
||||
from litellm.llms.dashscope.cost_calculator import (
|
||||
cost_per_token as dashscope_cost_per_token,
|
||||
|
|
@ -112,8 +111,19 @@ class TestDashscopeCostCalculator:
|
|||
|
||||
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_request_cache_control_is_explicit_mode_fallback(self):
|
||||
litellm.model_cost["dashscope/qwen-cache-control-fallback-test"] = {
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_tokens_details", "expected_cache_rate"),
|
||||
[
|
||||
({"cached_tokens": 600, "cache_type": "ephemeral"}, 1e-07),
|
||||
({"cached_tokens": 600}, 2e-07),
|
||||
],
|
||||
)
|
||||
def test_completion_cost_routes_dashscope_cache_mode_pricing(
|
||||
self,
|
||||
prompt_tokens_details: dict[str, int | str],
|
||||
expected_cache_rate: float,
|
||||
):
|
||||
litellm.model_cost["dashscope/qwen-completion-cache-mode-test"] = {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
|
|
@ -121,19 +131,27 @@ class TestDashscopeCostCalculator:
|
|||
"cache_read_input_token_cost": 1e-07,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details={"cached_tokens": 600},
|
||||
response = litellm.ModelResponse(
|
||||
model="qwen-completion-cache-mode-test",
|
||||
usage=Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=100,
|
||||
total_tokens=1100,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(
|
||||
model="qwen-cache-control-fallback-test",
|
||||
usage=usage,
|
||||
cache_control_requested=True,
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="qwen-completion-cache-mode-test",
|
||||
custom_llm_provider="dashscope",
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
|
||||
assert math.isclose(
|
||||
cost,
|
||||
(400 * 1e-06) + (600 * expected_cache_rate) + (100 * 4e-06),
|
||||
rel_tol=1e-10,
|
||||
)
|
||||
|
||||
def test_dashscope_tiered_pricing_uses_mode_specific_cache_read_rate(self):
|
||||
self._register_tiered_model(
|
||||
|
|
@ -158,7 +176,15 @@ class TestDashscopeCostCalculator:
|
|||
|
||||
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_cost_breakdown_uses_mode_specific_cache_read_rate(self):
|
||||
@pytest.mark.parametrize(
|
||||
("cache_type", "expected_cache_rate"),
|
||||
[("ephemeral", 1e-07), (None, 2e-07)],
|
||||
)
|
||||
def test_dashscope_cost_breakdown_matches_mode_specific_total(
|
||||
self,
|
||||
cache_type: str | None,
|
||||
expected_cache_rate: float,
|
||||
):
|
||||
litellm.model_cost["dashscope/qwen-cache-breakdown-mode-test"] = {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "chat",
|
||||
|
|
@ -167,87 +193,24 @@ class TestDashscopeCostCalculator:
|
|||
"cache_read_input_token_cost": 1e-07,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
}
|
||||
prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600}
|
||||
if cache_type is not None:
|
||||
prompt_tokens_details["cache_type"] = cache_type
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-breakdown-mode-test", usage=usage)
|
||||
breakdown = get_token_type_cost_breakdown(
|
||||
model="qwen-cache-breakdown-mode-test",
|
||||
custom_llm_provider="dashscope",
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert math.isclose(breakdown.cache_read_cost, 600 * 1e-07, rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_custom_pricing_uses_mode_specific_cache_read_rate(self):
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
|
||||
)
|
||||
|
||||
prompt_cost, _ = litellm_cost_per_token(
|
||||
model="custom-qwen",
|
||||
custom_llm_provider="dashscope",
|
||||
usage_object=usage,
|
||||
prompt_tokens=1000,
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
},
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10)
|
||||
|
||||
def test_completion_cost_detects_cache_control_request_fallback(self):
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=100,
|
||||
total_tokens=1100,
|
||||
prompt_tokens_details={"cached_tokens": 600},
|
||||
)
|
||||
response = litellm.ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="custom-qwen",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="custom-qwen",
|
||||
custom_llm_provider="dashscope",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "stable prefix",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"implicit_cache_read_input_token_cost": 2e-07,
|
||||
},
|
||||
)
|
||||
|
||||
assert math.isclose(
|
||||
cost,
|
||||
(400 * 1e-06) + (600 * 1e-07) + (100 * 4e-06),
|
||||
rel_tol=1e-10,
|
||||
)
|
||||
assert math.isclose(breakdown.cache_read_cost, 600 * expected_cache_rate, rel_tol=1e-10)
|
||||
assert math.isclose(prompt_cost, (400 * 1e-06) + breakdown.cache_read_cost, rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_tiered_pricing_within_first_tier(self):
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue