From 4db0efe3c759110db20995ad788dbf3bf1aed63d Mon Sep 17 00:00:00 2001 From: Marty Sullivan Date: Mon, 7 Sep 2026 04:30:13 -0400 Subject: [PATCH 1/3] fix(pricing): correct cached-token fields on realtime cost-map entries azure/gpt-realtime-2 was the only member of the gpt-realtime-2 family priced on one side of its cached-audio meter. Azure publishes that meter as "gpt-realtime-2 Audio cd inp Gl 1M Tokens" at 0.4 per 1M and charges the same rate for the write that populates the cache and the read that hits it, so cache_creation_input_audio_token_cost lands at 4e-07, matching azure/gpt-realtime-2.1, azure/gpt-realtime-2.1-mini and the openai gpt-realtime-2 entry. No cost path reads that field yet, so this corrects what get_model_info reports rather than what anything bills. The gemini Live entries go the other way. Google's Vertex context-caching page publishes separate supported-model lists for implicit and explicit caching, and no Live or native-audio model is in either one. Its pricing page prints N/A in both cached-input columns for every Gemini 2.5 Flash Live API row, where plain 2.5 Flash and 2.5 Flash-Lite both carry real cached prices, and the Vertex model card for the family marks context caching not supported outright. Vertex never reports cachedContentTokenCount on a Live session either, including for a byte-identical 7,021-token prefix replayed across sessions minutes apart, which is well past the 2,048-token minimum the same page sets for the Gemini 2 family. So the 7.5e-08 on the two preview siblings priced something the provider does not sell, and supports_prompt_caching on all three claimed a capability the model does not have. The rate comes out. The flag is set to false rather than removed, because get_model_info maps an absent key to None, and None is how this map spells "nobody checked" across the 2,788 entries that omit it, where false records the vendor's documented no. Both readers of the flag gate on `is True`, so nothing bills or behaves differently either way. Only the cached fields change on the two 09-2025 preview entries. Their source field points at the Gemini API pricing page rather than the Vertex one, so they describe a different surface with its own published limits, and their context windows are left alone rather than assumed to match the Vertex model card that drives the GA entry. Tests cover all three halves: the family invariant that a cached audio read implies an equal cached audio write, a cached count on a Live entry leaving the bill at the fresh-input total instead of adding the old 7.5e-08, and supports_prompt_caching answering false for all three entries while still answering true for 2.5 Flash, so the false cannot be a swallowed lookup error. --- ...odel_prices_and_context_window_backup.json | 9 +-- model_prices_and_context_window.json | 9 +-- tests/test_litellm/test_cost_calculator.py | 77 ++++++++++++++++++- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6c6e65927f9..4c2428b5e15 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5422,6 +5422,7 @@ "supports_tool_choice": true }, "azure/gpt-realtime-2": { + "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2026-08-31", @@ -23855,7 +23856,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -23871,7 +23872,6 @@ "input_cost_per_image_token": 3e-06 }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -23900,7 +23900,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -23915,7 +23915,6 @@ "gemini_native_audio": true }, "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -23945,7 +23944,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6c6e65927f9..4c2428b5e15 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5422,6 +5422,7 @@ "supports_tool_choice": true }, "azure/gpt-realtime-2": { + "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2026-08-31", @@ -23855,7 +23856,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -23871,7 +23872,6 @@ "input_cost_per_image_token": 3e-06 }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -23900,7 +23900,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -23915,7 +23915,6 @@ "gemini_native_audio": true }, "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -23945,7 +23944,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 1945e5ffac5..d25a08f6da8 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) -from litellm.utils import TranscriptionResponse +from litellm.utils import TranscriptionResponse, supports_prompt_caching @pytest.fixture @@ -288,6 +288,81 @@ def test_github_copilot_mai_code_1_flash_pricing(_local_model_cost_map, model): assert completion_usd == pytest.approx(500 * 4.5e-06) +GPT_REALTIME_2_FAMILY: Final = ( + "azure/gpt-realtime-2", + "azure/gpt-realtime-2.1", + "azure/gpt-realtime-2.1-mini", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", +) + + +def test_gpt_realtime_2_family_prices_audio_cache_writes_and_reads_alike(_local_model_cost_map: None) -> None: + """Azure publishes one cached-audio meter per gpt-realtime-2 deployment, charged at the same rate for + the write that populates the cache and the read that hits it. azure/gpt-realtime-2 carried only the + read side, so it was the one family member reporting no cache-creation audio price for a deployment + whose meter publishes one.""" + audio_cache_rates: Final = { + model: ( + litellm.model_cost[model].get("cache_read_input_audio_token_cost"), + litellm.model_cost[model].get("cache_creation_input_audio_token_cost"), + ) + for model in GPT_REALTIME_2_FAMILY + } + + assert all(read is not None and write == read for read, write in audio_cache_rates.values()), audio_cache_rates + assert audio_cache_rates["azure/gpt-realtime-2"] == (4e-07, 4e-07) + + +GEMINI_LIVE_NATIVE_AUDIO_CASES: Final = ( + ("gemini-live-2.5-flash-native-audio", "vertex_ai"), + ("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"), + ("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"), +) + + +@pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES) +def test_gemini_live_native_audio_carries_no_cached_input_rate( + _local_model_cost_map: None, model: str, provider: str +) -> None: + """Google publishes no cached-input price for the Live API. Its pricing table prints N/A in both + cached columns for every Gemini 2.5 Flash Live API row, and no Live or native-audio model appears + under either implicit or explicit context caching. Two of these entries priced a cached read at + 7.5e-08 regardless, which billed 0.008 here. With no invented rate the cached tokens drop out of + the bill, which is inert in practice because Vertex reports no cachedContentTokenCount on a Live + session.""" + assert litellm.get_model_info(model, custom_llm_provider=provider)["cache_read_input_token_cost"] is None + + prompt_usd, _ = cost_per_token( + model=model, + prompt_tokens=101_000, + completion_tokens=0, + custom_llm_provider=provider, + usage_object=Usage( + prompt_tokens=101_000, + completion_tokens=0, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000), + ), + ) + + assert prompt_usd == pytest.approx(1_000 * 5e-07) + + +@pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES) +def test_gemini_live_native_audio_declares_prompt_caching_unsupported( + _local_model_cost_map: None, model: str, provider: str +) -> None: + """The capability claim is what made the absent cached rate read as a pricing gap rather than a + vendor limitation. The flag has to say False rather than go missing: get_model_info maps an absent + key to None, which is how this map spells "nobody checked", where False records the vendor's + documented no. The 2.5 Flash control is load-bearing because supports_prompt_caching turns any + lookup error into False, so without it a broken lookup would read as a pass.""" + assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_prompt_caching"] is False + assert supports_prompt_caching(model=model, custom_llm_provider=provider) is False + assert supports_prompt_caching(model="gemini-2.5-flash", custom_llm_provider="vertex_ai") is True + + def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): usage = Usage( From 9058a84192e1e4fab2f7da27b7a2ccb24a6b989d Mon Sep 17 00:00:00 2001 From: Marty Sullivan Date: Mon, 7 Sep 2026 04:33:28 -0400 Subject: [PATCH 2/3] fix(cost): correct gemini-live-2.5-flash-native-audio limits and capabilities Google's model card for model ID gemini-live-2.5-flash-native-audio gives a 128K context window and 64K maximum output tokens, and marks structured output, context caching and URL context as not supported. Its modality list is text in and out, image in, audio in and out, and video in, with no document input of any kind. The entry advertised a 1M context window, an off-by-one 65535 output cap, and three capability flags the vendor marks unsupported. Context caching is the fourth and is handled in the cached-fields change alongside its two preview siblings. Both the bare id and vertex_ai/gemini-live-2.5-flash-native-audio resolve to this single entry, so the test drives the corrected values through both. --- ...odel_prices_and_context_window_backup.json | 12 ++++----- model_prices_and_context_window.json | 12 ++++----- tests/test_litellm/test_cost_calculator.py | 25 +++++++++++++++++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4c2428b5e15..1faadebc4e6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23830,9 +23830,9 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -23855,12 +23855,12 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_pdf_input": true, + "supports_pdf_input": false, "supports_prompt_caching": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_url_context": true, + "supports_url_context": false, "supports_vision": true, "supports_web_search": true, "search_context_cost_per_query": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4c2428b5e15..1faadebc4e6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23830,9 +23830,9 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -23855,12 +23855,12 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_pdf_input": true, + "supports_pdf_input": false, "supports_prompt_caching": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_url_context": true, + "supports_url_context": false, "supports_vision": true, "supports_web_search": true, "search_context_cost_per_query": { diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index d25a08f6da8..4c440524431 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4453,6 +4453,31 @@ def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) assert cost == pytest.approx(expected_cost, rel=1e-9) +@pytest.mark.parametrize( + "model", + ["gemini-live-2.5-flash-native-audio", "vertex_ai/gemini-live-2.5-flash-native-audio"], +) +def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_card( + _local_model_cost_map: None, model: str +) -> None: + """Google's card for model ID gemini-live-2.5-flash-native-audio gives a 128K context window and + 64K maximum output tokens, and marks structured output and URL context as not supported. Its + modality list is text, image, audio and video, with no document input, so pdf input cannot be + advertised either. The entry claimed a 1M window, an off-by-one 65535 output cap, and all three + capabilities. + + Both ids resolve to the one bare entry, which is why no vertex_ai/-prefixed twin is needed. + """ + info = litellm.get_model_info(model) + + assert info["max_input_tokens"] == 131072 + assert info["max_output_tokens"] == 65536 + assert info["max_tokens"] == 65536 + assert info["supports_response_schema"] is False + assert info["supports_url_context"] is False + assert info["supports_pdf_input"] is False + + @pytest.mark.parametrize( "priceless_entry", [ From 82c626e25d28d3e099eb6ca5a0fee639d6348f1b Mon Sep 17 00:00:00 2001 From: Marty Sullivan Date: Mon, 7 Sep 2026 06:29:36 -0400 Subject: [PATCH 3/3] test(cost): trim the docstrings on the realtime cost-map regressions Each of the four tests this PR added carried several lines of rationale for why the map was wrong. Cut them to one line saying what the test pins, and keep the two facts that explain a number rather than the change: the cached tokens dropping out of the bill, and why the 2.5 Flash control has to be there, both now assertion messages where they fire --- tests/test_litellm/test_cost_calculator.py | 33 ++++++---------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 4c440524431..04da42bce6f 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -299,10 +299,7 @@ GPT_REALTIME_2_FAMILY: Final = ( def test_gpt_realtime_2_family_prices_audio_cache_writes_and_reads_alike(_local_model_cost_map: None) -> None: - """Azure publishes one cached-audio meter per gpt-realtime-2 deployment, charged at the same rate for - the write that populates the cache and the read that hits it. azure/gpt-realtime-2 carried only the - read side, so it was the one family member reporting no cache-creation audio price for a deployment - whose meter publishes one.""" + """Azure publishes one cached-audio meter per gpt-realtime-2 deployment, so the write rate equals the read.""" audio_cache_rates: Final = { model: ( litellm.model_cost[model].get("cache_read_input_audio_token_cost"), @@ -326,12 +323,7 @@ GEMINI_LIVE_NATIVE_AUDIO_CASES: Final = ( def test_gemini_live_native_audio_carries_no_cached_input_rate( _local_model_cost_map: None, model: str, provider: str ) -> None: - """Google publishes no cached-input price for the Live API. Its pricing table prints N/A in both - cached columns for every Gemini 2.5 Flash Live API row, and no Live or native-audio model appears - under either implicit or explicit context caching. Two of these entries priced a cached read at - 7.5e-08 regardless, which billed 0.008 here. With no invented rate the cached tokens drop out of - the bill, which is inert in practice because Vertex reports no cachedContentTokenCount on a Live - session.""" + """Google prints N/A in both cached columns for every Live API row, so no cached-input rate can be charged.""" assert litellm.get_model_info(model, custom_llm_provider=provider)["cache_read_input_token_cost"] is None prompt_usd, _ = cost_per_token( @@ -346,21 +338,19 @@ def test_gemini_live_native_audio_carries_no_cached_input_rate( ), ) - assert prompt_usd == pytest.approx(1_000 * 5e-07) + assert prompt_usd == pytest.approx(1_000 * 5e-07), "the 100k cached tokens drop out with no cached rate to charge" @pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES) def test_gemini_live_native_audio_declares_prompt_caching_unsupported( _local_model_cost_map: None, model: str, provider: str ) -> None: - """The capability claim is what made the absent cached rate read as a pricing gap rather than a - vendor limitation. The flag has to say False rather than go missing: get_model_info maps an absent - key to None, which is how this map spells "nobody checked", where False records the vendor's - documented no. The 2.5 Flash control is load-bearing because supports_prompt_caching turns any - lookup error into False, so without it a broken lookup would read as a pass.""" + """The vendor's documented no has to be recorded as False, since an absent key reads back as None.""" assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_prompt_caching"] is False assert supports_prompt_caching(model=model, custom_llm_provider=provider) is False - assert supports_prompt_caching(model="gemini-2.5-flash", custom_llm_provider="vertex_ai") is True + assert supports_prompt_caching(model="gemini-2.5-flash", custom_llm_provider="vertex_ai") is True, ( + "control: the helper swallows a lookup error into False, so without this a broken lookup reads as a pass" + ) def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): @@ -4460,14 +4450,7 @@ def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_card( _local_model_cost_map: None, model: str ) -> None: - """Google's card for model ID gemini-live-2.5-flash-native-audio gives a 128K context window and - 64K maximum output tokens, and marks structured output and URL context as not supported. Its - modality list is text, image, audio and video, with no document input, so pdf input cannot be - advertised either. The entry claimed a 1M window, an off-by-one 65535 output cap, and all three - capabilities. - - Both ids resolve to the one bare entry, which is why no vertex_ai/-prefixed twin is needed. - """ + """Google's card for model ID gemini-live-2.5-flash-native-audio is the source for these limits and flags.""" info = litellm.get_model_info(model) assert info["max_input_tokens"] == 131072