diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ce632bc0eef..1e42e3ba9f4 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -75,6 +75,7 @@ from litellm.types.llms.vertex_ai import ( VertexToolName, ) from litellm.types.utils import ( + CachedTokensDetails, ChatCompletionAudioResponse, ChatCompletionTokenLogprob, ChoiceLogprobs, @@ -2380,8 +2381,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # Subtract from text tokens since implicit caching is primarily for text content # See: https://github.com/BerriAI/litellm/issues/16341 prompt_text_tokens = prompt_text_tokens - cached_tokens - if cached_audio_tokens is not None and prompt_audio_tokens is not None: - prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens if cached_image_tokens is not None and prompt_image_tokens is not None: prompt_image_tokens = prompt_image_tokens - cached_image_tokens if cached_video_tokens is not None and prompt_video_tokens is not None: @@ -2412,6 +2411,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): image_tokens=prompt_image_tokens, video_tokens=prompt_video_tokens, tool_use_tokens=tool_use_prompt_tokens, + cached_tokens_details=CachedTokensDetails(audio_tokens=cached_audio_tokens) + if cached_audio_tokens + else None, ) completion_tokens = response_tokens or completion_response["usageMetadata"].get("candidatesTokenCount", 0) diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 4c3ee9e7ca5..a9e7a86728f 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,5 +1,6 @@ import asyncio, importlib, os, uuid import json +import math import logging import re import threading @@ -6098,6 +6099,34 @@ def test_vertex_ai_usage_metadata_video_tokens_with_caching(): assert result.prompt_tokens_details.audio_tokens == 200 +def test_gemini_cached_audio_tokens_billed_at_audio_cache_read_rate(): + usage: Final = VertexGeminiConfig().calculate_usage( + completion_response={ + "usageMetadata": { + "promptTokenCount": 10000, + "candidatesTokenCount": 10, + "totalTokenCount": 10010, + "cachedContentTokenCount": 4000, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 6000}, + {"modality": "AUDIO", "tokenCount": 4000}, + ], + "cacheTokensDetails": [{"modality": "AUDIO", "tokenCount": 4000}], + } + } + ) + + model_info: Final = litellm.get_model_info(model="gemini-2.5-flash", custom_llm_provider="gemini") + input_rate: Final = model_info.get("input_cost_per_token") + audio_cache_read_rate: Final = model_info.get("cache_read_input_audio_token_cost") + assert input_rate is not None and audio_cache_read_rate is not None + assert audio_cache_read_rate != model_info.get("cache_read_input_token_cost") + prompt_cost, _ = litellm.cost_per_token(model="gemini-2.5-flash", custom_llm_provider="gemini", usage_object=usage) + assert math.isclose(prompt_cost, 6000 * input_rate + 4000 * audio_cache_read_rate) + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.audio_tokens == 4000 + + def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt(): """Test promptTokensDetails with DOCUMENT modality for PDF inputs.