scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(cost): carry cache_read_input_audio_token_cost through get_model_info

Browse source 
Every proxy and router cost lookup goes through get_model_info, which copies
cost map keys explicitly, so the new audio cache-read branch always fell back
to the text cache-read rate there. Copy the key so models whose audio
cache-read rate differs from the text one bill cached audio correctly.
This commit is contained in:
mateo-berri 2026-09-12 15:31:53 -07:00
parent 3c2342bfd3
commit e4b0588362
4 changed files with 27 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/types/utils.py
View file

@ -251,6 +251,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing

1
litellm/utils.py
View file

@ -5866,6 +5866,7 @@ def _get_model_info_helper(
"cache_creation_input_token_cost_ultrafast", None
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens", None

17
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -5235,3 +5235,20 @@ def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7)
def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_lookup(_local_model_cost_map: None) -> None:
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=400,
audio_tokens=600,
cached_tokens=500,
cached_tokens_details={"text_tokens": 100, "audio_tokens": 400},
),
)
prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai")
assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7)

8
tests/test_litellm/test_utils.py
View file

@ -6430,3 +6430,11 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
assert snapshot["litellm_call_id"]
assert snapshot["response_cost"] is not None
assert snapshot["api_base"]
def test_get_model_info_carries_cache_read_input_audio_token_cost(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
info = litellm.get_model_info("gpt-realtime-2.1-mini", custom_llm_provider="openai")
assert info["cache_read_input_audio_token_cost"] == 3e-07
assert info["cache_read_input_token_cost"] == 6e-08