scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix(cost): activate long-context cache tiers without input-tier gate; cover Responses dict path

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
milan 2026-07-23 19:22:36 +00:00
parent afb1e134ae
commit 6a2a7a3930
3 changed files with 76 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -39,6 +39,13 @@ _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency)
# of being rebuilt for every model_info key on every call.
_SERVICE_TIER_SUFFIXES: tuple[str, ...] = tuple(f"_{st.value}" for st in ServiceTier)
_ABOVE_THRESHOLD_KEY_PREFIXES: tuple[str, ...] = (
"input_cost_per_token_above_",
"output_cost_per_token_above_",
"cache_creation_input_token_cost_above_",
"cache_read_input_token_cost_above_",
)
def _get_token_detail_value(details: object, key: str) -> Optional[int]:
if isinstance(details, dict):
@ -239,7 +246,12 @@ def _get_token_base_cost(
# so that the threshold detection loop only processes standard keys. The
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
threshold_keys = [
k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES)
k
for k in model_info
if k.startswith(_ABOVE_THRESHOLD_KEY_PREFIXES)
and k.endswith("_tokens")
and "_above_1hr" not in k
and not k.endswith(_SERVICE_TIER_SUFFIXES)
]
if not threshold_keys:
return (
@ -270,7 +282,7 @@ def _get_token_base_cost(
service_tier,
)
if service_tier
else key
else f"input_cost_per_token_above_{threshold_str}_tokens"
)
prompt_base_cost = cast(
float,

44
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -1016,6 +1016,50 @@ def test_generic_cost_per_token_prices_cache_write_at_above_272k_tier():
assert prompt_cost == pytest.approx(expected_prompt_cost, rel=1e-9)
def test_generic_cost_per_token_activates_cache_creation_tier_without_input_tier():
"""Regression for LIT-4725 / #33772: a long-context cache-creation tier must
activate on its own. When only cache_creation_input_token_cost_above_272k_tokens
is configured (no matching input_cost_per_token_above_272k_tokens), threshold
discovery must still find the tier so cache-write tokens past 272k bill at the
tiered rate while plain input stays at the base rate."""
model = "litellm-test-cache-write-272k-cache-only"
custom_llm_provider = "openai"
litellm.register_model(
{
model: {
"litellm_provider": custom_llm_provider,
"mode": "chat",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 4e-6,
"cache_creation_input_token_cost": 1.25e-6,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-6,
}
}
)
plain_tokens, cache_write = 200_000, 100_000
prompt_tokens_details = PromptTokensDetailsWrapper()
prompt_tokens_details.cache_write_tokens = cache_write
usage = Usage(
prompt_tokens=plain_tokens + cache_write,
completion_tokens=100,
total_tokens=plain_tokens + cache_write + 100,
prompt_tokens_details=prompt_tokens_details,
)
try:
prompt_cost, _ = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider,
)
finally:
litellm.model_cost.pop(model, None)
expected_prompt_cost = plain_tokens * 1e-6 + cache_write * 2.5e-6
assert prompt_cost == pytest.approx(expected_prompt_cost, rel=1e-9)
def test_string_cost_values():
"""Test that cost values defined as strings are properly converted to floats."""
from unittest.mock import patch

31
tests/test_litellm/responses/test_responses_utils.py
View file

@ -464,13 +464,16 @@ class TestResponseAPILoggingUtils:
def test_transform_response_api_usage_maps_cache_write_tokens_dict(self):
"""Regression for LIT-4725 / #33772: the Responses API reports cache writes under
input_tokens_details.cache_write_tokens. The chat-shaped usage must carry them on
cache_creation_tokens so cost is computed identically to /chat/completions."""
usage = {
"input_tokens": 10_000,
"output_tokens": 20,
"total_tokens": 10_020,
"input_tokens_details": {"cached_tokens": 2_000, "cache_write_tokens": 8_000},
}
cache_creation_tokens so cost is computed identically to /chat/completions.
model_construct keeps input_tokens_details a raw dict so the dict branch runs."""
from litellm.types.llms.openai import ResponseAPIUsage
usage = ResponseAPIUsage.model_construct(
input_tokens=10_000,
output_tokens=20,
total_tokens=10_020,
input_tokens_details={"cached_tokens": 2_000, "cache_write_tokens": 8_000},
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
@ -482,12 +485,14 @@ class TestResponseAPILoggingUtils:
def test_transform_response_api_usage_cache_creation_tokens_precedence_dict(self):
"""When both cache_creation_tokens and cache_write_tokens are present, the explicit
cache_creation_tokens wins (they describe the same tokens under different names)."""
usage = {
"input_tokens": 10_000,
"output_tokens": 20,
"total_tokens": 10_020,
"input_tokens_details": {"cache_creation_tokens": 5_000, "cache_write_tokens": 8_000},
}
from litellm.types.llms.openai import ResponseAPIUsage
usage = ResponseAPIUsage.model_construct(
input_tokens=10_000,
output_tokens=20,
total_tokens=10_020,
input_tokens_details={"cache_creation_tokens": 5_000, "cache_write_tokens": 8_000},
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)