mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(perplexity): stop double-billing reasoning tokens in manual cost fallback (#30488)
* fix(perplexity): stop double-billing reasoning tokens in manual cost fallback When perplexity_cost_per_token cannot use the API-provided usage.cost.total_cost short-circuit and falls back to manual calculation, it multiplies the full usage.completion_tokens by output_cost_per_token and then adds reasoning_tokens * output_cost_per_reasoning_token on top. Per the OpenAI/Perplexity usage convention codified for the central path in PR #18607, completion_tokens already INCLUDES reasoning_tokens, so the manual fallback double-bills reasoning at both the output and reasoning rate. Concrete impact on perplexity/sonar-deep-research (input 2e-6, output 8e-6, reasoning 3e-6): for the exact usage shape exercised by the live response fixture in tests/llm_translation/test_perplexity_reasoning.py (prompt_tokens=9, completion_tokens=20, reasoning_tokens=15) the current code charges 0.000223 vs the convention-correct 0.000103, a 2.165x overcharge. The bug is reachable whenever Perplexity omits the cost object (streaming chunks, fixture-driven paths, older API versions). Subtracts reasoning_tokens (clamped at zero) from completion_tokens before applying the output rate, mirroring how dashscope/cost_calculator.py and the central generic_cost_per_token already handle it. Preserves the existing fallback behaviour when output_cost_per_reasoning_token is unset (all completion_tokens stay at the output rate). Existing tests in tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py asserted the buggy math and are updated to the convention-correct math. Adds a focused regression test using the exact usage shape from the live response fixture so this class of bug cannot be silently reintroduced. * style(perplexity): drop redundant type annotation on else branch to satisfy mypy mypy [no-redef] flagged 'completion_cost' as declared in both if and else arms; keeping the annotation only on the first declaration matches existing patterns in this file. * fix(perplexity): update integration test expected costs for non-double-billed math Three tests in test_perplexity_integration.py asserted the old buggy expectation that reasoning_tokens are billed in addition to the full completion_tokens count. After the fix in cost_per_token, reasoning_tokens are billed at the reasoning rate and the remaining (completion_tokens - reasoning_tokens) at the standard output rate, matching OpenAI/Perplexity convention (PR #18607). Updates: test_end_to_end_cost_calculation_with_transformation, test_main_cost_calculator_integration, test_high_volume_cost_calculation. The high-volume sanity threshold drops to 0.25 to reflect the corrected total.
This commit is contained in:
Anuj ojha
GitHub
parent
d4915766d6
commit
5ac0c57383
3 changed files with 89 additions and 48 deletions
|
|
@ -58,11 +58,8 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
|
|||
|
||||
## CALCULATE OUTPUT COST
|
||||
output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token"))
|
||||
completion_cost: float = (usage.completion_tokens or 0) * output_cost_per_token
|
||||
|
||||
## ADD REASONING TOKENS COST (if present)
|
||||
reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0
|
||||
# Also check completion_tokens_details if reasoning_tokens is not directly available
|
||||
if (
|
||||
reasoning_tokens == 0
|
||||
and hasattr(usage, "completion_tokens_details")
|
||||
|
|
@ -73,9 +70,19 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
|
|||
)
|
||||
|
||||
reasoning_cost_value = model_info.get("output_cost_per_reasoning_token")
|
||||
|
||||
# `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage
|
||||
# convention (codified for the central path in PR #18607). When a reasoning rate is
|
||||
# configured we subtract before the output-rate multiplication so the reasoning
|
||||
# tokens are not billed twice.
|
||||
if reasoning_tokens > 0 and reasoning_cost_value is not None:
|
||||
reasoning_cost_per_token = _safe_float_cast(reasoning_cost_value)
|
||||
completion_cost += reasoning_tokens * reasoning_cost_per_token
|
||||
non_reasoning_completion_tokens = max(
|
||||
0, (usage.completion_tokens or 0) - reasoning_tokens
|
||||
)
|
||||
completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token
|
||||
completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value)
|
||||
else:
|
||||
completion_cost = (usage.completion_tokens or 0) * output_cost_per_token
|
||||
|
||||
## ADD SEARCH QUERIES COST (if present)
|
||||
num_search_queries = 0
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Test file for Perplexity cost calculator functionality.
|
||||
|
||||
Tests the cost calculation for Perplexity models including citation tokens,
|
||||
Tests the cost calculation for Perplexity models including citation tokens,
|
||||
search queries, and reasoning tokens.
|
||||
"""
|
||||
|
||||
|
|
@ -21,7 +21,11 @@ from litellm.cost_calculator import completion_cost, cost_per_token
|
|||
from litellm.llms.perplexity.cost_calculator import (
|
||||
cost_per_token as perplexity_cost_per_token,
|
||||
)
|
||||
from litellm.types.utils import Usage, PromptTokensDetailsWrapper
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
Usage,
|
||||
PromptTokensDetailsWrapper,
|
||||
)
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
|
|
@ -135,13 +139,14 @@ class TestPerplexityCostCalculator:
|
|||
model="sonar-deep-research", usage=usage
|
||||
)
|
||||
|
||||
# Expected costs:
|
||||
# Input: 100 tokens * $2e-6 = $0.0002
|
||||
# Output: 50 tokens * $8e-6 = $0.0004
|
||||
# Reasoning: 20 tokens * $3e-6 = $0.00006
|
||||
# Total completion cost: $0.00046
|
||||
# `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity
|
||||
# convention codified in PR #18607. Non-reasoning portion = 50 - 20 = 30.
|
||||
# Input: 100 tokens * $2e-6 = $0.0002
|
||||
# Output (text): 30 tokens * $8e-6 = $0.00024
|
||||
# Reasoning: 20 tokens * $3e-6 = $0.00006
|
||||
# Total completion cost = $0.0003
|
||||
expected_prompt_cost = 100 * 2e-6
|
||||
expected_completion_cost = (50 * 8e-6) + (20 * 3e-6)
|
||||
expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
|
||||
|
|
@ -159,13 +164,10 @@ class TestPerplexityCostCalculator:
|
|||
model="sonar-deep-research", usage=usage
|
||||
)
|
||||
|
||||
# Expected costs:
|
||||
# Input: 100 tokens * $2e-6 = $0.0002
|
||||
# Output: 50 tokens * $8e-6 = $0.0004
|
||||
# Reasoning: 20 tokens * $3e-6 = $0.00006
|
||||
# Total completion cost: $0.00046
|
||||
# Same convention as the direct-attribute case above; reasoning is a subset of
|
||||
# completion_tokens, so non-reasoning portion = 50 - 20 = 30.
|
||||
expected_prompt_cost = 100 * 2e-6
|
||||
expected_completion_cost = (50 * 8e-6) + (20 * 3e-6)
|
||||
expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
|
||||
|
|
@ -187,16 +189,16 @@ class TestPerplexityCostCalculator:
|
|||
model="sonar-deep-research", usage=usage
|
||||
)
|
||||
|
||||
# Expected costs:
|
||||
# Input: 100 tokens * $2e-6 = $0.0002
|
||||
# Citation: 30 tokens * $2e-6 = $0.00006
|
||||
# Total prompt cost: $0.00026
|
||||
# Output: 50 tokens * $8e-6 = $0.0004
|
||||
# Reasoning: 15 tokens * $3e-6 = $0.000045
|
||||
# Search: 2 queries * ($0.005 / 1000) = $0.00001
|
||||
# Total completion cost: $0.000455
|
||||
# Expected costs (reasoning is a subset of completion_tokens):
|
||||
# Input: 100 tokens * $2e-6 = $0.0002
|
||||
# Citation: 30 tokens * $2e-6 = $0.00006
|
||||
# Total prompt cost = $0.00026
|
||||
# Output (text): (50 - 15) tokens * $8e-6 = $0.00028
|
||||
# Reasoning: 15 tokens * $3e-6 = $0.000045
|
||||
# Search: 2 queries * ($0.005 / 1000) = $0.00001
|
||||
# Total completion cost = $0.000335
|
||||
expected_prompt_cost = (100 * 2e-6) + (30 * 2e-6)
|
||||
expected_completion_cost = (50 * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005)
|
||||
expected_completion_cost = ((50 - 15) * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
|
||||
|
|
@ -306,11 +308,11 @@ class TestPerplexityCostCalculator:
|
|||
completion_response=response, custom_llm_provider="perplexity"
|
||||
)
|
||||
|
||||
# Calculate expected total cost
|
||||
# Calculate expected total cost (reasoning is a subset of completion_tokens)
|
||||
expected_prompt_cost = (100 * 2e-6) + (15 * 2e-6) # Input + citation
|
||||
expected_completion_cost = (
|
||||
(50 * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005)
|
||||
) # Output + reasoning + search
|
||||
((50 - 10) * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005)
|
||||
) # Output (text) + reasoning + search
|
||||
expected_total = expected_prompt_cost + expected_completion_cost
|
||||
|
||||
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
|
||||
|
|
@ -353,10 +355,13 @@ class TestPerplexityCostCalculator:
|
|||
model="sonar-deep-research", usage=usage
|
||||
)
|
||||
|
||||
# Calculate expected costs
|
||||
# Calculate expected costs. `completion_tokens` includes `reasoning_tokens`,
|
||||
# so non-reasoning portion = 50 - reasoning_tokens.
|
||||
expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6)
|
||||
expected_completion_cost = (
|
||||
(50 * 8e-6) + (reasoning_tokens * 3e-6) + (search_queries / 1000 * 0.005)
|
||||
((50 - reasoning_tokens) * 8e-6)
|
||||
+ (reasoning_tokens * 3e-6)
|
||||
+ (search_queries / 1000 * 0.005)
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
|
||||
|
|
@ -413,3 +418,36 @@ class TestPerplexityCostCalculator:
|
|||
|
||||
assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-6)
|
||||
assert math.isclose(completion_cost, expected_completion, rel_tol=1e-6)
|
||||
|
||||
def test_reasoning_tokens_not_double_billed(self):
|
||||
"""
|
||||
Regression: `completion_tokens` includes `reasoning_tokens` per the
|
||||
OpenAI/Perplexity usage convention (codified for the central path in PR #18607).
|
||||
When `output_cost_per_reasoning_token` is configured the manual fallback must
|
||||
subtract reasoning from completion before applying the output rate so the
|
||||
reasoning tokens are not billed at BOTH the output rate and the reasoning rate.
|
||||
|
||||
Uses the exact usage shape produced by the live response fixture in
|
||||
`tests/llm_translation/test_perplexity_reasoning.py`.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=9,
|
||||
completion_tokens=20,
|
||||
total_tokens=29,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
reasoning_tokens=15
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = perplexity_cost_per_token(
|
||||
model="sonar-deep-research", usage=usage
|
||||
)
|
||||
|
||||
# sonar-deep-research rates: input 2e-6, output 8e-6, reasoning 3e-6.
|
||||
# Non-reasoning portion of the 20 completion tokens = 20 - 15 = 5.
|
||||
# Pre-fix this asserted 20 * 8e-6 + 15 * 3e-6 = 2.05e-4 (a 2.16x overcharge).
|
||||
expected_prompt = 9 * 2e-6
|
||||
expected_completion = (20 - 15) * 8e-6 + 15 * 3e-6
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-9)
|
||||
assert math.isclose(completion_cost, expected_completion, rel_tol=1e-9)
|
||||
|
|
|
|||
|
|
@ -104,12 +104,10 @@ class TestPerplexityIntegration:
|
|||
)
|
||||
citation_tokens = citation_chars // 4
|
||||
|
||||
expected_prompt_cost = (100 * 2e-6) + (
|
||||
citation_tokens * 2e-6
|
||||
) # Input + citation
|
||||
expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6)
|
||||
expected_completion_cost = (
|
||||
(50 * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005)
|
||||
) # Output + reasoning + search
|
||||
((50 - 10) * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005)
|
||||
)
|
||||
expected_total = expected_prompt_cost + expected_completion_cost
|
||||
|
||||
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
|
||||
|
|
@ -152,11 +150,10 @@ class TestPerplexityIntegration:
|
|||
usage_object=usage,
|
||||
)
|
||||
|
||||
# Calculate expected costs
|
||||
expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6) # Input + citation
|
||||
expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6)
|
||||
expected_completion_cost = (
|
||||
(100 * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005)
|
||||
) # Output + reasoning + search
|
||||
((100 - 25) * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005)
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
|
||||
assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6)
|
||||
|
|
@ -263,15 +260,14 @@ class TestPerplexityIntegration:
|
|||
custom_llm_provider="perplexity",
|
||||
)
|
||||
|
||||
# Calculate expected cost
|
||||
expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6) # $0.11
|
||||
expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6)
|
||||
expected_completion_cost = (
|
||||
(25000 * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005)
|
||||
) # $0.23
|
||||
expected_total = expected_prompt_cost + expected_completion_cost # $0.34
|
||||
((25000 - 10000) * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005)
|
||||
)
|
||||
expected_total = expected_prompt_cost + expected_completion_cost
|
||||
|
||||
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
|
||||
assert total_cost > 0.3 # Sanity check for high-volume scenario
|
||||
assert total_cost > 0.25
|
||||
|
||||
def test_transformation_preserves_existing_usage_fields(self):
|
||||
"""Test that transformation doesn't overwrite existing standard usage fields."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue