mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix: price cached tokens in custom cost calculator
This commit is contained in:
gabrielzirondi
parent
295a36aa69
commit
9f2ef352a5
3 changed files with 658 additions and 75 deletions
|
|
@ -25,9 +25,12 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
CostCalculatorUtils,
|
||||
_generic_cost_per_character,
|
||||
_get_service_tier_cost_key,
|
||||
_parse_prompt_tokens_details,
|
||||
_get_prompt_tokens_details_for_cost_calc,
|
||||
_get_token_base_cost,
|
||||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_token,
|
||||
generic_cost_per_token_from_model_info,
|
||||
get_billable_input_tokens,
|
||||
select_cost_metric_for_model,
|
||||
)
|
||||
|
|
@ -173,6 +176,10 @@ def _cost_per_token_custom_pricing_helper(
|
|||
prompt_tokens: float = 0,
|
||||
completion_tokens: float = 0,
|
||||
response_time_ms: Optional[float] = 0.0,
|
||||
usage_object: Optional[Usage] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
service_tier: Optional[str] = None,
|
||||
custom_cost_model_info: Optional[ModelInfo] = None,
|
||||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: Optional[CostPerToken] = None,
|
||||
custom_cost_per_second: Optional[float] = None,
|
||||
|
|
@ -182,6 +189,22 @@ def _cost_per_token_custom_pricing_helper(
|
|||
return None
|
||||
|
||||
if custom_cost_per_token is not None:
|
||||
if usage_object is not None and _custom_cost_per_token_has_cache_pricing(
|
||||
custom_cost_per_token
|
||||
):
|
||||
model_info = (
|
||||
custom_cost_model_info
|
||||
or _get_model_info_from_custom_cost_per_token(
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
return generic_cost_per_token_from_model_info(
|
||||
model_info=model_info,
|
||||
usage=usage_object,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
input_cost = custom_cost_per_token["input_cost_per_token"] * prompt_tokens
|
||||
output_cost = custom_cost_per_token["output_cost_per_token"] * completion_tokens
|
||||
return input_cost, output_cost
|
||||
|
|
@ -192,6 +215,35 @@ def _cost_per_token_custom_pricing_helper(
|
|||
return None
|
||||
|
||||
|
||||
def _custom_cost_per_token_has_cache_pricing(
|
||||
custom_cost_per_token: CostPerToken,
|
||||
) -> bool:
|
||||
return any(
|
||||
key.startswith("cache_read_input_token_cost")
|
||||
or key.startswith("cache_creation_input_token_cost")
|
||||
for key in cast(dict, custom_cost_per_token)
|
||||
)
|
||||
|
||||
|
||||
def _get_model_info_from_custom_cost_per_token(
|
||||
custom_cost_per_token: CostPerToken,
|
||||
custom_llm_provider: Optional[str],
|
||||
) -> ModelInfo:
|
||||
model_info: dict[str, Any] = {
|
||||
"key": "custom_cost_per_token",
|
||||
"max_tokens": None,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": None,
|
||||
"input_cost_per_token": custom_cost_per_token["input_cost_per_token"],
|
||||
"output_cost_per_token": custom_cost_per_token["output_cost_per_token"],
|
||||
"litellm_provider": custom_llm_provider or "custom",
|
||||
"mode": "chat",
|
||||
"supported_openai_params": None,
|
||||
}
|
||||
model_info.update(cast(dict[str, Any], custom_cost_per_token))
|
||||
return cast(ModelInfo, model_info)
|
||||
|
||||
|
||||
def _get_additional_costs(
|
||||
model: str,
|
||||
custom_llm_provider: Optional[str],
|
||||
|
|
@ -272,6 +324,7 @@ def cost_per_token( # noqa: PLR0915
|
|||
cache_read_input_tokens: Optional[int] = 0,
|
||||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: Optional[CostPerToken] = None,
|
||||
custom_cost_model_info: Optional[ModelInfo] = None,
|
||||
custom_cost_per_second: Optional[float] = None,
|
||||
### NUMBER OF QUERIES ###
|
||||
number_of_queries: Optional[int] = None,
|
||||
|
|
@ -327,6 +380,10 @@ def cost_per_token( # noqa: PLR0915
|
|||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time_ms=response_time_ms,
|
||||
usage_object=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
custom_cost_model_info=custom_cost_model_info,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
)
|
||||
|
|
@ -1012,6 +1069,85 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
pass
|
||||
|
||||
|
||||
def _get_cache_costs_for_breakdown(
|
||||
usage_object: Optional[Usage],
|
||||
model: Optional[str],
|
||||
custom_llm_provider: Optional[str],
|
||||
custom_cost_per_token: Optional[CostPerToken],
|
||||
model_info: Optional[ModelInfo] = None,
|
||||
service_tier: Optional[str] = None,
|
||||
) -> Tuple[Optional[float], Optional[float]]:
|
||||
"""Return cache-read/write costs for standard logging breakdown fields."""
|
||||
if usage_object is None:
|
||||
return None, None
|
||||
|
||||
prompt_tokens_details = _get_prompt_tokens_details_for_cost_calc(usage_object)
|
||||
|
||||
if (
|
||||
prompt_tokens_details["cache_hit_tokens"] <= 0
|
||||
and prompt_tokens_details["cache_creation_tokens"] <= 0
|
||||
and prompt_tokens_details["cache_creation_token_details"] is None
|
||||
):
|
||||
return None, None
|
||||
|
||||
if (
|
||||
model_info is None
|
||||
and custom_cost_per_token is not None
|
||||
and _custom_cost_per_token_has_cache_pricing(custom_cost_per_token)
|
||||
):
|
||||
model_info = _get_model_info_from_custom_cost_per_token(
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
elif model_info is None and model:
|
||||
try:
|
||||
model_info = cast(
|
||||
ModelInfo,
|
||||
litellm.get_model_info(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
if model_info is None:
|
||||
return None, None
|
||||
|
||||
(
|
||||
_,
|
||||
_,
|
||||
cache_creation_cost_per_token,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost_per_token,
|
||||
) = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage_object,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
cache_read_cost = (
|
||||
prompt_tokens_details["cache_hit_tokens"] * cache_read_cost_per_token
|
||||
if prompt_tokens_details["cache_hit_tokens"] > 0
|
||||
else None
|
||||
)
|
||||
cache_creation_cost = (
|
||||
calculate_cache_writing_cost(
|
||||
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
|
||||
cache_creation_token_details=prompt_tokens_details[
|
||||
"cache_creation_token_details"
|
||||
],
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=cache_creation_cost_per_token,
|
||||
)
|
||||
if (
|
||||
prompt_tokens_details["cache_creation_tokens"] > 0
|
||||
or prompt_tokens_details["cache_creation_token_details"] is not None
|
||||
)
|
||||
else None
|
||||
)
|
||||
return cache_read_cost, cache_creation_cost
|
||||
|
||||
|
||||
def completion_cost( # noqa: PLR0915
|
||||
completion_response=None,
|
||||
model: Optional[str] = None,
|
||||
|
|
@ -1501,6 +1637,16 @@ def completion_cost( # noqa: PLR0915
|
|||
if litellm_logging_obj is not None:
|
||||
request_model_for_cost = litellm_logging_obj.model
|
||||
|
||||
custom_cost_model_info: Optional[ModelInfo] = None
|
||||
if (
|
||||
custom_cost_per_token is not None
|
||||
and _custom_cost_per_token_has_cache_pricing(custom_cost_per_token)
|
||||
):
|
||||
custom_cost_model_info = _get_model_info_from_custom_cost_per_token(
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
(
|
||||
prompt_tokens_cost_usd_dollar,
|
||||
completion_tokens_cost_usd_dollar,
|
||||
|
|
@ -1513,6 +1659,7 @@ def completion_cost( # noqa: PLR0915
|
|||
region_name=region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_model_info=custom_cost_model_info,
|
||||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
|
|
@ -1602,37 +1749,26 @@ def completion_cost( # noqa: PLR0915
|
|||
|
||||
# Store cost breakdown in logging object if available
|
||||
if litellm_logging_obj is not None:
|
||||
_cache_read_cost: Optional[float] = None
|
||||
_cache_creation_cost: Optional[float] = None
|
||||
if cost_per_token_usage_object is not None:
|
||||
_cr = getattr(
|
||||
cost_per_token_usage_object, "cache_read_input_tokens", None
|
||||
) or (cost_per_token_usage_object.model_extra or {}).get(
|
||||
"cache_read_input_tokens"
|
||||
_cache_read_cost, _cache_creation_cost = (
|
||||
_get_cache_costs_for_breakdown(
|
||||
usage_object=cost_per_token_usage_object,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
model_info=custom_cost_model_info,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
_cc = getattr(
|
||||
cost_per_token_usage_object,
|
||||
"cache_creation_input_tokens",
|
||||
None,
|
||||
) or (cost_per_token_usage_object.model_extra or {}).get(
|
||||
"cache_creation_input_tokens"
|
||||
)
|
||||
if (_cr or _cc) and model:
|
||||
try:
|
||||
_mi = litellm.get_model_info(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
_cr_rate = _mi.get("cache_read_input_token_cost")
|
||||
if _cr and _cr_rate is not None:
|
||||
_cache_read_cost = float(_cr) * float(_cr_rate)
|
||||
_cc_rate = _mi.get("cache_creation_input_token_cost")
|
||||
if _cc and _cc_rate is not None:
|
||||
_cache_creation_cost = float(_cc) * float(_cc_rate)
|
||||
except Exception:
|
||||
pass
|
||||
)
|
||||
_prompt_tokens_cost_for_breakdown = (
|
||||
prompt_tokens_cost_usd_dollar
|
||||
- (_cache_read_cost or 0.0)
|
||||
- (_cache_creation_cost or 0.0)
|
||||
)
|
||||
if _prompt_tokens_cost_for_breakdown < 0:
|
||||
_prompt_tokens_cost_for_breakdown = 0.0
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
|
||||
prompt_tokens_cost_usd_dollar=_prompt_tokens_cost_for_breakdown,
|
||||
completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar,
|
||||
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools,
|
||||
total_cost_usd_dollar=_final_cost,
|
||||
|
|
@ -2136,7 +2272,7 @@ def batch_cost_calculator(
|
|||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
# Add cache read cost if applicable
|
||||
details = _parse_prompt_tokens_details(usage)
|
||||
details = _get_prompt_tokens_details_for_cost_calc(usage)
|
||||
cache_read_tokens = details["cache_hit_tokens"]
|
||||
cache_read_cost_key = _get_service_tier_cost_key(
|
||||
"cache_read_input_token_cost", None
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ def get_billable_input_tokens(usage: Usage) -> int:
|
|||
Returns the number of billable input tokens.
|
||||
Subtracts cached tokens from prompt tokens if applicable.
|
||||
"""
|
||||
details = _parse_prompt_tokens_details(usage)
|
||||
details = _get_prompt_tokens_details_for_cost_calc(usage)
|
||||
return usage.prompt_tokens - details["cache_hit_tokens"]
|
||||
|
||||
|
||||
|
|
@ -270,41 +270,43 @@ def _get_token_base_cost(
|
|||
)
|
||||
|
||||
# Apply tiered pricing to cache costs
|
||||
cache_creation_tiered_key = (
|
||||
f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
||||
cache_creation_tiered_key = _get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
cache_creation_1hr_tiered_key = f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
|
||||
cache_read_tiered_key = (
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens"
|
||||
cache_creation_1hr_tiered_key = _get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
cache_read_tiered_key = _get_service_tier_cost_key(
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
|
||||
if cache_creation_tiered_key in model_info:
|
||||
cache_creation_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_tiered_key,
|
||||
cache_creation_cost,
|
||||
),
|
||||
)
|
||||
cache_creation_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_tiered_key,
|
||||
cache_creation_cost,
|
||||
),
|
||||
)
|
||||
|
||||
if cache_creation_1hr_tiered_key in model_info:
|
||||
cache_creation_cost_above_1hr = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_1hr_tiered_key,
|
||||
cache_creation_cost_above_1hr,
|
||||
),
|
||||
)
|
||||
cache_creation_cost_above_1hr = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_1hr_tiered_key,
|
||||
cache_creation_cost_above_1hr,
|
||||
),
|
||||
)
|
||||
|
||||
if cache_read_tiered_key in model_info:
|
||||
cache_read_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info, cache_read_tiered_key, cache_read_cost
|
||||
),
|
||||
)
|
||||
cache_read_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info, cache_read_tiered_key, cache_read_cost
|
||||
),
|
||||
)
|
||||
|
||||
break
|
||||
except (IndexError, ValueError):
|
||||
|
|
@ -496,6 +498,57 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
|
|||
)
|
||||
|
||||
|
||||
def _get_usage_cache_token_count(usage: Usage, field_name: str) -> int:
|
||||
value = getattr(usage, field_name, None)
|
||||
if value is None:
|
||||
value = (usage.model_extra or {}).get(field_name)
|
||||
return int(value or 0)
|
||||
|
||||
|
||||
def _get_prompt_tokens_details_for_cost_calc(
|
||||
usage: Usage,
|
||||
) -> PromptTokensDetailsResult:
|
||||
top_level_cache_read_tokens = _get_usage_cache_token_count(
|
||||
usage, "cache_read_input_tokens"
|
||||
)
|
||||
top_level_cache_creation_tokens = _get_usage_cache_token_count(
|
||||
usage, "cache_creation_input_tokens"
|
||||
)
|
||||
|
||||
if usage.prompt_tokens_details is not None:
|
||||
prompt_tokens_details = _parse_prompt_tokens_details(usage)
|
||||
if (
|
||||
prompt_tokens_details["cache_hit_tokens"] <= 0
|
||||
and top_level_cache_read_tokens > 0
|
||||
):
|
||||
prompt_tokens_details["cache_hit_tokens"] = top_level_cache_read_tokens
|
||||
if (
|
||||
prompt_tokens_details["cache_creation_tokens"] <= 0
|
||||
and top_level_cache_creation_tokens > 0
|
||||
):
|
||||
prompt_tokens_details["cache_creation_tokens"] = (
|
||||
top_level_cache_creation_tokens
|
||||
)
|
||||
return prompt_tokens_details
|
||||
|
||||
return PromptTokensDetailsResult(
|
||||
cache_hit_tokens=top_level_cache_read_tokens,
|
||||
cache_creation_tokens=top_level_cache_creation_tokens,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=max(
|
||||
0,
|
||||
usage.prompt_tokens
|
||||
- top_level_cache_read_tokens
|
||||
- top_level_cache_creation_tokens,
|
||||
),
|
||||
audio_tokens=0,
|
||||
image_tokens=0,
|
||||
character_count=0,
|
||||
image_count=0,
|
||||
video_length_seconds=0.0,
|
||||
)
|
||||
|
||||
|
||||
class CompletionTokensDetailsResult(TypedDict):
|
||||
audio_tokens: int
|
||||
text_tokens: int
|
||||
|
|
@ -639,23 +692,29 @@ def generic_cost_per_token( # noqa: PLR0915
|
|||
## GET MODEL INFO
|
||||
model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
return generic_cost_per_token_from_model_info(
|
||||
model_info=model_info, usage=usage, service_tier=service_tier
|
||||
)
|
||||
|
||||
|
||||
def generic_cost_per_token_from_model_info( # noqa: PLR0915
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
service_tier: Optional[str] = None,
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Calculates token costs from an already resolved model_info object.
|
||||
|
||||
This shares the same pricing behavior as generic_cost_per_token(), including
|
||||
prompt cache pricing, service tier keys, threshold pricing, and modality
|
||||
token costs.
|
||||
"""
|
||||
|
||||
## CALCULATE INPUT COST
|
||||
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
|
||||
prompt_cost = 0.0
|
||||
### PROCESSING COST
|
||||
prompt_tokens_details = PromptTokensDetailsResult(
|
||||
cache_hit_tokens=0,
|
||||
cache_creation_tokens=0,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=usage.prompt_tokens,
|
||||
audio_tokens=0,
|
||||
image_tokens=0,
|
||||
character_count=0,
|
||||
image_count=0,
|
||||
video_length_seconds=0.0,
|
||||
)
|
||||
if usage.prompt_tokens_details:
|
||||
prompt_tokens_details = _parse_prompt_tokens_details(usage)
|
||||
prompt_tokens_details = _get_prompt_tokens_details_for_cost_calc(usage)
|
||||
|
||||
## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
|
||||
## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,13 @@ from litellm.cost_calculator import (
|
|||
response_cost_calculator,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIRealtimeStreamList
|
||||
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
|
||||
from litellm.types.utils import (
|
||||
CacheCreationTokenDetails,
|
||||
CompletionTokensDetailsWrapper,
|
||||
ModelResponse,
|
||||
PromptTokensDetailsWrapper,
|
||||
Usage,
|
||||
)
|
||||
from litellm.utils import TranscriptionResponse
|
||||
|
||||
|
||||
|
|
@ -1106,6 +1112,388 @@ def test_azure_ai_cache_cost_calculation():
|
|||
), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}"
|
||||
|
||||
|
||||
def _gpt_5_4_cached_usage() -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=6074,
|
||||
completion_tokens=285,
|
||||
total_tokens=6359,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=None,
|
||||
audio_tokens=0,
|
||||
image_tokens=None,
|
||||
cached_tokens=3456,
|
||||
),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
text_tokens=None,
|
||||
audio_tokens=0,
|
||||
image_tokens=None,
|
||||
reasoning_tokens=0,
|
||||
accepted_prediction_tokens=0,
|
||||
rejected_prediction_tokens=0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _gpt_5_4_custom_cost_per_token():
|
||||
return {
|
||||
"input_cost_per_token": 2.5e-6,
|
||||
"output_cost_per_token": 1.5e-5,
|
||||
"cache_read_input_token_cost": 2.5e-7,
|
||||
}
|
||||
|
||||
|
||||
def _expected_gpt_5_4_cached_costs():
|
||||
raw_input_cost = (6074 - 3456) * 2.5e-6
|
||||
cache_read_cost = 3456 * 2.5e-7
|
||||
output_cost = 285 * 1.5e-5
|
||||
return raw_input_cost, cache_read_cost, output_cost
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,custom_llm_provider",
|
||||
[
|
||||
("openai/gpt-5.4", "openai"),
|
||||
("custom_openai/openai/gpt-5.4", "custom_openai"),
|
||||
],
|
||||
)
|
||||
def test_completion_cost_custom_cost_per_token_uses_cached_token_pricing(
|
||||
model, custom_llm_provider
|
||||
):
|
||||
"""
|
||||
Explicit custom token pricing must not bill cached input tokens at the
|
||||
regular input rate. This covers both OpenAI-format names seen in proxy
|
||||
chains: the public model name and the custom_openai/provider-prefixed key.
|
||||
"""
|
||||
usage = _gpt_5_4_cached_usage()
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model=model,
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
custom_cost_per_token=_gpt_5_4_custom_cost_per_token(),
|
||||
)
|
||||
|
||||
raw_input_cost, cache_read_cost, output_cost = _expected_gpt_5_4_cached_costs()
|
||||
assert result_cost == pytest.approx(raw_input_cost + cache_read_cost + output_cost)
|
||||
|
||||
|
||||
def test_completion_cost_custom_cost_per_token_uses_flex_cache_pricing():
|
||||
usage = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=10,
|
||||
total_tokens=110,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40),
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 10e-6,
|
||||
"cache_read_input_token_cost": 0.5e-6,
|
||||
"input_cost_per_token_flex": 0.2e-6,
|
||||
"output_cost_per_token_flex": 2e-6,
|
||||
"cache_read_input_token_cost_flex": 0.1e-6,
|
||||
},
|
||||
service_tier="flex",
|
||||
)
|
||||
|
||||
expected_cost = 60 * 0.2e-6 + 40 * 0.1e-6 + 10 * 2e-6
|
||||
assert result_cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_completion_cost_custom_cost_per_token_uses_above_threshold_cache_pricing():
|
||||
usage = Usage(
|
||||
prompt_tokens=210000,
|
||||
completion_tokens=100,
|
||||
total_tokens=210100,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=10000),
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 0.1e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 3e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 4e-6,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 0.3e-6,
|
||||
},
|
||||
)
|
||||
|
||||
expected_cost = 200000 * 3e-6 + 10000 * 0.3e-6 + 100 * 4e-6
|
||||
assert result_cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_completion_cost_custom_cost_per_token_uses_priority_above_threshold_cache_pricing():
|
||||
usage = Usage(
|
||||
prompt_tokens=210000,
|
||||
completion_tokens=100,
|
||||
total_tokens=210100,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=10000),
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 0.1e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 3e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 4e-6,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 0.3e-6,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 5e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 6e-6,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 0.9e-6,
|
||||
},
|
||||
service_tier="priority",
|
||||
)
|
||||
|
||||
expected_cost = 200000 * 5e-6 + 10000 * 0.9e-6 + 100 * 6e-6
|
||||
assert result_cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_completion_cost_custom_cost_per_token_uses_priority_cache_creation_details():
|
||||
usage = Usage(
|
||||
prompt_tokens=150,
|
||||
completion_tokens=10,
|
||||
total_tokens=160,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=50,
|
||||
cached_tokens=20,
|
||||
cache_creation_tokens=80,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=30,
|
||||
ephemeral_1h_input_tokens=50,
|
||||
),
|
||||
),
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 0.1e-6,
|
||||
"cache_creation_input_token_cost": 1.5e-6,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-6,
|
||||
"input_cost_per_token_priority": 3e-6,
|
||||
"output_cost_per_token_priority": 4e-6,
|
||||
"cache_read_input_token_cost_priority": 0.3e-6,
|
||||
"cache_creation_input_token_cost_priority": 3.5e-6,
|
||||
},
|
||||
service_tier="priority",
|
||||
)
|
||||
|
||||
expected_cost = 50 * 3e-6 + 20 * 0.3e-6 + 30 * 3.5e-6 + 50 * 2e-6 + 10 * 4e-6
|
||||
assert result_cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,custom_llm_provider,registered_key",
|
||||
[
|
||||
("openai/gpt-5.4", "openai", "openai/gpt-5.4"),
|
||||
(
|
||||
"custom_openai/openai/gpt-5.4",
|
||||
"custom_openai",
|
||||
"custom_openai/openai/gpt-5.4",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_completion_cost_model_map_pricing_uses_cached_tokens_without_custom_cost_per_token(
|
||||
model, custom_llm_provider, registered_key
|
||||
):
|
||||
"""
|
||||
The regular model-cost-map path already applies cache-read pricing without
|
||||
passing completion_cost(custom_cost_per_token=...).
|
||||
"""
|
||||
litellm.register_model(
|
||||
{
|
||||
registered_key: {
|
||||
"key": registered_key,
|
||||
"input_cost_per_token": 2.5e-6,
|
||||
"output_cost_per_token": 1.5e-5,
|
||||
"cache_read_input_token_cost": 2.5e-7,
|
||||
"litellm_provider": custom_llm_provider,
|
||||
"mode": "chat",
|
||||
}
|
||||
}
|
||||
)
|
||||
usage = _gpt_5_4_cached_usage()
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model=model,
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
raw_input_cost, cache_read_cost, output_cost = _expected_gpt_5_4_cached_costs()
|
||||
assert result_cost == pytest.approx(raw_input_cost + cache_read_cost + output_cost)
|
||||
|
||||
|
||||
def test_completion_cost_breakdown_splits_cache_read_cost_for_custom_openai_model_map():
|
||||
"""
|
||||
Proxy logging should expose raw input cost and cache-read cost separately.
|
||||
The total should still include both components.
|
||||
"""
|
||||
model = "openai/gpt-5.4"
|
||||
litellm.register_model(
|
||||
{
|
||||
"custom_openai/openai/gpt-5.4": {
|
||||
"key": "custom_openai/openai/gpt-5.4",
|
||||
"input_cost_per_token": 2.5e-6,
|
||||
"output_cost_per_token": 1.5e-5,
|
||||
"cache_read_input_token_cost": 2.5e-7,
|
||||
"litellm_provider": "custom_openai",
|
||||
"mode": "chat",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
usage = _gpt_5_4_cached_usage()
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model=model,
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
class MockLoggingObj:
|
||||
def __init__(self):
|
||||
self.model = model
|
||||
self.cost_breakdown = None
|
||||
|
||||
def set_cost_breakdown(self, **kwargs):
|
||||
self.cost_breakdown = kwargs
|
||||
|
||||
logging_obj = MockLoggingObj()
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="custom_openai",
|
||||
custom_pricing=True,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
raw_input_cost, cache_read_cost, output_cost = _expected_gpt_5_4_cached_costs()
|
||||
assert result_cost == pytest.approx(raw_input_cost + cache_read_cost + output_cost)
|
||||
assert logging_obj.cost_breakdown["input_cost"] == pytest.approx(raw_input_cost)
|
||||
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(
|
||||
cache_read_cost
|
||||
)
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(output_cost)
|
||||
|
||||
|
||||
def test_completion_cost_custom_cost_per_token_uses_top_level_cache_tokens_without_prompt_details():
|
||||
"""
|
||||
Top-level cache token fields should drive both billing and logging breakdown
|
||||
when prompt_tokens_details is absent.
|
||||
"""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110)
|
||||
usage.prompt_tokens_details = None
|
||||
usage.cache_read_input_tokens = 40
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
class MockLoggingObj:
|
||||
def __init__(self):
|
||||
self.model = "openai/gpt-5.4"
|
||||
self.cost_breakdown = None
|
||||
|
||||
def set_cost_breakdown(self, **kwargs):
|
||||
self.cost_breakdown = kwargs
|
||||
|
||||
logging_obj = MockLoggingObj()
|
||||
|
||||
result_cost = completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 0.1e-6,
|
||||
},
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
raw_input_cost = 60 * 1e-6
|
||||
cache_read_cost = 40 * 0.1e-6
|
||||
output_cost = 10 * 2e-6
|
||||
assert result_cost == pytest.approx(raw_input_cost + cache_read_cost + output_cost)
|
||||
assert logging_obj.cost_breakdown["input_cost"] == pytest.approx(raw_input_cost)
|
||||
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(
|
||||
cache_read_cost
|
||||
)
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(output_cost)
|
||||
|
||||
|
||||
def test_cost_discount_vertex_ai():
|
||||
"""
|
||||
Test that cost discount is applied correctly for Vertex AI provider
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue