scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-17 23:51:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 66

fix(cost): price OpenAI cache_write_tokens and register tiered cache-creation costs

Browse source
This commit is contained in:
Devin AI 2026-07-17 21:11:43 +00:00
parent 71e0251341
commit 909f402ed4
6 changed files with 198 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -461,6 +461,8 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
)
or 0
)
if not cache_creation_tokens:
cache_creation_tokens = _coerce_token_count(getattr(usage.prompt_tokens_details, "cache_write_tokens", 0))
cache_creation_token_details = (
cast(
Optional[CacheCreationTokenDetails],

4
litellm/responses/utils.py
View file

@ -993,6 +993,10 @@ class ResponseAPILoggingUtils:
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
cache_creation_tokens=(
getattr(response_api_usage.input_tokens_details, "cache_creation_tokens", None)
or getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None)
),
)
completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None
output_tokens_details = getattr(response_api_usage, "output_tokens_details", None)

6
litellm/types/utils.py
View file

@ -187,7 +187,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing
input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing
cache_creation_input_token_cost: Optional[float]
cache_creation_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing
cache_creation_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing
cache_creation_input_token_cost_above_200k_tokens: Optional[float]
cache_creation_input_token_cost_above_272k_tokens: Optional[float]
cache_creation_input_token_cost_above_1hr: Optional[float]
cache_read_input_token_cost: Optional[float]
cache_read_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing
@ -3038,8 +3041,11 @@ class CustomPricingLiteLLMParams(BaseModel):
input_cost_per_token_flex: Optional[float] = None
input_cost_per_token_priority: Optional[float] = None
cache_creation_input_token_cost: Optional[float] = None
cache_creation_input_token_cost_flex: Optional[float] = None
cache_creation_input_token_cost_priority: Optional[float] = None
cache_creation_input_token_cost_above_1hr: Optional[float] = None
cache_creation_input_token_cost_above_200k_tokens: Optional[float] = None
cache_creation_input_token_cost_above_272k_tokens: Optional[float] = None
cache_creation_input_audio_token_cost: Optional[float] = None
cache_read_input_token_cost: Optional[float] = None
cache_read_input_token_cost_flex: Optional[float] = None

10
litellm/utils.py
View file

@ -2611,8 +2611,11 @@ _BEDROCK_REGION_PREFIXES = (
_CACHE_PRICING_FIELDS = (
"cache_creation_input_token_cost",
"cache_creation_input_token_cost_flex",
"cache_creation_input_token_cost_priority",
"cache_creation_input_token_cost_above_1hr",
"cache_creation_input_token_cost_above_200k_tokens",
"cache_creation_input_token_cost_above_272k_tokens",
"cache_read_input_token_cost",
"cache_read_input_token_cost_above_200k_tokens",
)
@ -5399,9 +5402,16 @@ def _get_model_info_helper(
input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None),
input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None),
cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None),
cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None),
cache_creation_input_token_cost_priority=_model_info.get(
"cache_creation_input_token_cost_priority", None
),
cache_creation_input_token_cost_above_200k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_200k_tokens", None
),
cache_creation_input_token_cost_above_272k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens", None
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(

127
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -2237,3 +2237,130 @@ def test_token_type_cost_breakdown_applies_regional_uplift():
text_input_cost = 600 * model_info["input_cost_per_token"] * uplift
assert text_output_cost + eu.reasoning_cost == pytest.approx(completion_cost)
assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost)
def test_generic_cost_per_token_prices_cache_write_tokens_at_cache_creation_rate():
"""
Regression for https://github.com/BerriAI/litellm/issues/33772.
OpenAI(-compatible) providers report cache-write tokens under
`prompt_tokens_details.cache_write_tokens`, not the Anthropic-style
`cache_creation_tokens`. The total-cost path (generic_cost_per_token ->
_parse_prompt_tokens_details) previously ignored `cache_write_tokens`, so those
tokens were billed at the plain input rate instead of the cache-creation rate.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
model = "gpt-5.6"
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=0, cache_write_tokens=400
),
)
prompt_cost, _ = generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="openai"
)
model_info = litellm.get_model_info(model=model, custom_llm_provider="openai")
expected = (
600 * model_info["input_cost_per_token"]
+ 400 * model_info["cache_creation_input_token_cost"]
)
assert prompt_cost == pytest.approx(expected)
# Guard against regressing to the plain-input-rate mispricing the bug described.
assert prompt_cost != pytest.approx(1000 * model_info["input_cost_per_token"])
def test_get_model_info_registers_tiered_cache_creation_cost_keys():
"""
Regression for https://github.com/BerriAI/litellm/issues/33772.
get_model_info previously dropped the tiered cache-creation cost keys, so even
when a model config defined them they never reached ModelInfo and cache-write
cost could not vary by service/context tier.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
model = "gpt-5.6"
model_info = litellm.get_model_info(model=model, custom_llm_provider="openai")
raw = litellm.model_cost[model]
for key in (
"cache_creation_input_token_cost_flex",
"cache_creation_input_token_cost_priority",
"cache_creation_input_token_cost_above_272k_tokens",
):
assert model_info[key] == raw[key]
def test_generic_cost_per_token_applies_priority_cache_creation_rate():
"""
Regression for https://github.com/BerriAI/litellm/issues/33772.
With the tiered keys registered, a `priority` service tier must price cache-write
tokens at cache_creation_input_token_cost_priority, not the standard rate.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
model = "gpt-5.6"
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=0, cache_write_tokens=400
),
)
prompt_cost, _ = generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="openai", service_tier="priority"
)
model_info = litellm.get_model_info(model=model, custom_llm_provider="openai")
expected = (
600 * model_info["input_cost_per_token_priority"]
+ 400 * model_info["cache_creation_input_token_cost_priority"]
)
assert prompt_cost == pytest.approx(expected)
def test_generic_cost_per_token_applies_above_272k_cache_creation_rate():
"""
Regression for https://github.com/BerriAI/litellm/issues/33772.
Prompts above the 272k threshold must price cache-write tokens at
cache_creation_input_token_cost_above_272k_tokens.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
model = "gpt-5.6"
cache_write = 400
prompt_tokens = 300_000
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=0,
total_tokens=prompt_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=0, cache_write_tokens=cache_write
),
)
prompt_cost, _ = generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="openai"
)
model_info = litellm.get_model_info(model=model, custom_llm_provider="openai")
text_tokens = prompt_tokens - cache_write
expected = (
text_tokens * model_info["input_cost_per_token_above_272k_tokens"]
+ cache_write * model_info["cache_creation_input_token_cost_above_272k_tokens"]
)
assert prompt_cost == pytest.approx(expected)

49
tests/test_litellm/responses/test_responses_utils.py
View file

@ -240,6 +240,55 @@ class TestResponseAPILoggingUtils:
and result.prompt_tokens_details.cached_tokens == 2
)
def test_transform_response_api_usage_carries_cache_write_tokens_from_object(self):
"""
Regression for https://github.com/BerriAI/litellm/issues/33772.
When input_tokens_details arrives as a ResponseAPIUsage object (not a dict),
the OpenAI(-compatible) `cache_write_tokens` split was dropped, so cost was
computed without any cache-write tokens. It must be carried through onto the
chat-shaped `cache_creation_tokens`.
"""
from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage
usage = ResponseAPIUsage(
input_tokens=1000,
output_tokens=0,
total_tokens=1000,
input_tokens_details=InputTokensDetails(
cached_tokens=0, cache_write_tokens=400
),
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cache_creation_tokens == 400
def test_transform_response_api_usage_carries_cache_write_tokens_from_dict(self):
"""
Regression for https://github.com/BerriAI/litellm/issues/33772.
The dict-shaped usage path must also carry the `cache_write_tokens` split
through onto `cache_creation_tokens` so the downstream cost path can price it
at the cache-creation rate.
"""
usage = {
"input_tokens": 1000,
"output_tokens": 0,
"total_tokens": 1000,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 400},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cache_creation_tokens == 400
def test_transform_response_api_usage_with_none_values(self):
"""Test transformation handles None values properly"""
# Setup