mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(pricing): drop the unpublished cached rate from the Gemini Live preview entries (#42651)
* fix(pricing): correct cached-token fields on realtime cost-map entries azure/gpt-realtime-2 was the only member of the gpt-realtime-2 family priced on one side of its cached-audio meter. Azure publishes that meter as "gpt-realtime-2 Audio cd inp Gl 1M Tokens" at 0.4 per 1M and charges the same rate for the write that populates the cache and the read that hits it, so cache_creation_input_audio_token_cost lands at 4e-07, matching azure/gpt-realtime-2.1, azure/gpt-realtime-2.1-mini and the openai gpt-realtime-2 entry. No cost path reads that field yet, so this corrects what get_model_info reports rather than what anything bills. The gemini Live entries go the other way. Google's Vertex context-caching page publishes separate supported-model lists for implicit and explicit caching, and no Live or native-audio model is in either one. Its pricing page prints N/A in both cached-input columns for every Gemini 2.5 Flash Live API row, where plain 2.5 Flash and 2.5 Flash-Lite both carry real cached prices, and the Vertex model card for the family marks context caching not supported outright. Vertex never reports cachedContentTokenCount on a Live session either, including for a byte-identical 7,021-token prefix replayed across sessions minutes apart, which is well past the 2,048-token minimum the same page sets for the Gemini 2 family. So the 7.5e-08 on the two preview siblings priced something the provider does not sell, and supports_prompt_caching on all three claimed a capability the model does not have. The rate comes out. The flag is set to false rather than removed, because get_model_info maps an absent key to None, and None is how this map spells "nobody checked" across the 2,788 entries that omit it, where false records the vendor's documented no. Both readers of the flag gate on `is True`, so nothing bills or behaves differently either way. Only the cached fields change on the two 09-2025 preview entries. Their source field points at the Gemini API pricing page rather than the Vertex one, so they describe a different surface with its own published limits, and their context windows are left alone rather than assumed to match the Vertex model card that drives the GA entry. Tests cover all three halves: the family invariant that a cached audio read implies an equal cached audio write, a cached count on a Live entry leaving the bill at the fresh-input total instead of adding the old 7.5e-08, and supports_prompt_caching answering false for all three entries while still answering true for 2.5 Flash, so the false cannot be a swallowed lookup error. * fix(cost): correct gemini-live-2.5-flash-native-audio limits and capabilities Google's model card for model ID gemini-live-2.5-flash-native-audio gives a 128K context window and 64K maximum output tokens, and marks structured output, context caching and URL context as not supported. Its modality list is text in and out, image in, audio in and out, and video in, with no document input of any kind. The entry advertised a 1M context window, an off-by-one 65535 output cap, and three capability flags the vendor marks unsupported. Context caching is the fourth and is handled in the cached-fields change alongside its two preview siblings. Both the bare id and vertex_ai/gemini-live-2.5-flash-native-audio resolve to this single entry, so the test drives the corrected values through both. * test(integration): cover live preview cached tokens billed at the fresh rate Co-authored-by: Marty Sullivan <marty@martysullivan.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): cite dated sources for Live entry pins and drop restating docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Marty Sullivan <marty@martysullivan.com> Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
5dfaa8d620
commit
3eb7e45615
5 changed files with 155 additions and 8 deletions
|
|
@ -26075,7 +26075,6 @@
|
|||
"input_cost_per_image_token": 3e-06
|
||||
},
|
||||
"gemini-live-2.5-flash-preview-native-audio-09-2025": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -26104,7 +26103,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -26120,7 +26119,6 @@
|
|||
"input_cost_per_image_token": 3e-06
|
||||
},
|
||||
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -26150,7 +26148,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
|
|||
|
|
@ -26075,7 +26075,6 @@
|
|||
"input_cost_per_image_token": 3e-06
|
||||
},
|
||||
"gemini-live-2.5-flash-preview-native-audio-09-2025": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -26104,7 +26103,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -26120,7 +26119,6 @@
|
|||
"input_cost_per_image_token": 3e-06
|
||||
},
|
||||
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -26150,7 +26148,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
|
|||
|
|
@ -290,6 +290,12 @@
|
|||
"tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [
|
||||
"quota_management.spend_tracking.spend_calculate.rejects_unpriced_model"
|
||||
],
|
||||
"tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": [
|
||||
"quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"
|
||||
],
|
||||
"tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": [
|
||||
"quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"
|
||||
],
|
||||
"tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [
|
||||
"mgmt.key.update.project_detach_denied_to_restricted_actor"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -18,3 +18,51 @@ def test_spend_calculate_rejects_unpriced_model_with_400(gateway: Gateway) -> No
|
|||
assert error["type"] == "invalid_request_error", response.text
|
||||
assert error["param"] == "model", response.text
|
||||
assert model in string_value(error["message"]), response.text
|
||||
|
||||
|
||||
GEMINI_LIVE_PREVIEW_MODELS: Final = (
|
||||
"gemini-live-2.5-flash-preview-native-audio-09-2025",
|
||||
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GEMINI_LIVE_PREVIEW_MODELS)
|
||||
@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate")
|
||||
def test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate(gateway: Gateway, model: str) -> None:
|
||||
def cost_with_cached_tokens(cached_tokens: int) -> float:
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/spend/calculate",
|
||||
{
|
||||
"completion_response": {
|
||||
"id": "chatcmpl-live-preview",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "live preview answer"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 101_000,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 101_000,
|
||||
"prompt_tokens_details": {"cached_tokens": cached_tokens},
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
cost: Final = object_value(JSON_OBJECT.validate_json(response.text))["cost"]
|
||||
assert isinstance(cost, int | float)
|
||||
return float(cost)
|
||||
|
||||
cached_cost: Final = cost_with_cached_tokens(100_000)
|
||||
fresh_cost: Final = cost_with_cached_tokens(0)
|
||||
assert fresh_cost > 0, fresh_cost
|
||||
assert cached_cost == pytest.approx(fresh_cost), (
|
||||
f"the entry publishes no cached rate, so 100k cached tokens must bill like fresh ones: {cached_cost} vs {fresh_cost}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from litellm.types.utils import (
|
|||
Usage,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -4536,3 +4537,99 @@ def test_cost_per_token_bedrock_nemotron_super_3_uses_eu_west_2_entry_not_us_rat
|
|||
|
||||
assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"])
|
||||
assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"])
|
||||
|
||||
|
||||
GPT_REALTIME_2_FAMILY: Final = (
|
||||
"azure/gpt-realtime-2.1",
|
||||
"azure/gpt-realtime-2.1-mini",
|
||||
"gpt-realtime-2",
|
||||
"gpt-realtime-2.1",
|
||||
"gpt-realtime-2.1-mini",
|
||||
)
|
||||
|
||||
|
||||
def test_gpt_realtime_2_family_prices_audio_cache_writes_and_reads_alike(_local_model_cost_map: None) -> None:
|
||||
audio_cache_rates: Final = {
|
||||
model: (
|
||||
litellm.model_cost[model].get("cache_read_input_audio_token_cost"),
|
||||
litellm.model_cost[model].get("cache_creation_input_audio_token_cost"),
|
||||
)
|
||||
for model in GPT_REALTIME_2_FAMILY
|
||||
}
|
||||
|
||||
# Azure publishes one cached-audio meter per gpt-realtime-2 deployment,
|
||||
# https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/, checked 2026-09-23
|
||||
assert all(read is not None and write == read for read, write in audio_cache_rates.values()), audio_cache_rates
|
||||
assert len(audio_cache_rates) == len(GPT_REALTIME_2_FAMILY)
|
||||
|
||||
|
||||
GEMINI_LIVE_NATIVE_AUDIO_CASES: Final = (
|
||||
("gemini-live-2.5-flash-native-audio", "vertex_ai"),
|
||||
("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"),
|
||||
("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES)
|
||||
def test_gemini_live_native_audio_carries_no_cached_input_rate(
|
||||
_local_model_cost_map: None, model: str, provider: str
|
||||
) -> None:
|
||||
# the Vertex pricing table prints N/A for cached input on every Live row,
|
||||
# https://cloud.google.com/vertex-ai/generative-ai/pricing, checked 2026-09-23
|
||||
assert litellm.get_model_info(model, custom_llm_provider=provider)["cache_read_input_token_cost"] is None
|
||||
|
||||
prompt_usd, _ = cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=101_000,
|
||||
completion_tokens=0,
|
||||
custom_llm_provider=provider,
|
||||
usage_object=Usage(
|
||||
prompt_tokens=101_000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000),
|
||||
),
|
||||
)
|
||||
fresh_usd, _ = cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=101_000,
|
||||
completion_tokens=0,
|
||||
custom_llm_provider=provider,
|
||||
usage_object=Usage(prompt_tokens=101_000, completion_tokens=0),
|
||||
)
|
||||
|
||||
assert prompt_usd == pytest.approx(fresh_usd), (
|
||||
"with no cached rate the cached tokens bill at the input rate, so a phantom discount cannot appear"
|
||||
)
|
||||
assert prompt_usd > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES)
|
||||
def test_gemini_live_native_audio_declares_prompt_caching_unsupported(
|
||||
_local_model_cost_map: None, model: str, provider: str
|
||||
) -> None:
|
||||
# the Vertex context-caching supported-model lists contain no Live model while 2.5 Flash is listed,
|
||||
# https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-overview, checked 2026-09-23
|
||||
assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_prompt_caching"] is False
|
||||
assert supports_prompt_caching(model=model, custom_llm_provider=provider) is False
|
||||
assert supports_prompt_caching(model="gemini-2.5-flash", custom_llm_provider="vertex_ai") is True, (
|
||||
"control: the helper swallows a lookup error into False, so without this a broken lookup reads as a pass"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["gemini-live-2.5-flash-native-audio", "vertex_ai/gemini-live-2.5-flash-native-audio"],
|
||||
)
|
||||
def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_card(
|
||||
_local_model_cost_map: None, model: str
|
||||
) -> None:
|
||||
info = litellm.get_model_info(model)
|
||||
|
||||
# the Vertex model card for gemini-live-2.5-flash-native-audio publishes these limits and flags,
|
||||
# https://cloud.google.com/vertex-ai/generative-ai/docs/models, checked 2026-09-23
|
||||
assert info["max_input_tokens"] == 131072
|
||||
assert info["max_output_tokens"] == 65536
|
||||
assert info["max_tokens"] == 65536
|
||||
assert info["supports_response_schema"] is False
|
||||
assert info["supports_url_context"] is False
|
||||
assert info["supports_pdf_input"] is False
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue