fix(gemini): bill cached audio tokens at the audio cache-read rate

This commit is contained in:
astroknot-sheep 2026-10-21 22:48:43 +00:00
parent b6f91a221b
commit 55eea8ac63
2 changed files with 33 additions and 2 deletions

View file

@ -75,6 +75,7 @@ from litellm.types.llms.vertex_ai import (
VertexToolName,
)
from litellm.types.utils import (
CachedTokensDetails,
ChatCompletionAudioResponse,
ChatCompletionTokenLogprob,
ChoiceLogprobs,
@ -2380,8 +2381,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# Subtract from text tokens since implicit caching is primarily for text content
# See: https://github.com/BerriAI/litellm/issues/16341
prompt_text_tokens = prompt_text_tokens - cached_tokens
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
if cached_image_tokens is not None and prompt_image_tokens is not None:
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
if cached_video_tokens is not None and prompt_video_tokens is not None:
@ -2412,6 +2411,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
image_tokens=prompt_image_tokens,
video_tokens=prompt_video_tokens,
tool_use_tokens=tool_use_prompt_tokens,
cached_tokens_details=CachedTokensDetails(audio_tokens=cached_audio_tokens)
if cached_audio_tokens
else None,
)
completion_tokens = response_tokens or completion_response["usageMetadata"].get("candidatesTokenCount", 0)

View file

@ -1,5 +1,6 @@
import asyncio, importlib, os, uuid
import json
import math
import logging
import re
import threading
@ -6098,6 +6099,34 @@ def test_vertex_ai_usage_metadata_video_tokens_with_caching():
assert result.prompt_tokens_details.audio_tokens == 200
def test_gemini_cached_audio_tokens_billed_at_audio_cache_read_rate():
usage: Final = VertexGeminiConfig().calculate_usage(
completion_response={
"usageMetadata": {
"promptTokenCount": 10000,
"candidatesTokenCount": 10,
"totalTokenCount": 10010,
"cachedContentTokenCount": 4000,
"promptTokensDetails": [
{"modality": "TEXT", "tokenCount": 6000},
{"modality": "AUDIO", "tokenCount": 4000},
],
"cacheTokensDetails": [{"modality": "AUDIO", "tokenCount": 4000}],
}
}
)
model_info: Final = litellm.get_model_info(model="gemini-2.5-flash", custom_llm_provider="gemini")
input_rate: Final = model_info.get("input_cost_per_token")
audio_cache_read_rate: Final = model_info.get("cache_read_input_audio_token_cost")
assert input_rate is not None and audio_cache_read_rate is not None
assert audio_cache_read_rate != model_info.get("cache_read_input_token_cost")
prompt_cost, _ = litellm.cost_per_token(model="gemini-2.5-flash", custom_llm_provider="gemini", usage_object=usage)
assert math.isclose(prompt_cost, 6000 * input_rate + 4000 * audio_cache_read_rate)
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.audio_tokens == 4000
def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt():
"""Test promptTokensDetails with DOCUMENT modality for PDF inputs.