mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(gemini): bill cached audio tokens at the audio cache-read rate
This commit is contained in:
parent
b6f91a221b
commit
55eea8ac63
2 changed files with 33 additions and 2 deletions
|
|
@ -75,6 +75,7 @@ from litellm.types.llms.vertex_ai import (
|
|||
VertexToolName,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
CachedTokensDetails,
|
||||
ChatCompletionAudioResponse,
|
||||
ChatCompletionTokenLogprob,
|
||||
ChoiceLogprobs,
|
||||
|
|
@ -2380,8 +2381,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# Subtract from text tokens since implicit caching is primarily for text content
|
||||
# See: https://github.com/BerriAI/litellm/issues/16341
|
||||
prompt_text_tokens = prompt_text_tokens - cached_tokens
|
||||
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
|
||||
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
|
||||
if cached_image_tokens is not None and prompt_image_tokens is not None:
|
||||
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
|
||||
if cached_video_tokens is not None and prompt_video_tokens is not None:
|
||||
|
|
@ -2412,6 +2411,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
image_tokens=prompt_image_tokens,
|
||||
video_tokens=prompt_video_tokens,
|
||||
tool_use_tokens=tool_use_prompt_tokens,
|
||||
cached_tokens_details=CachedTokensDetails(audio_tokens=cached_audio_tokens)
|
||||
if cached_audio_tokens
|
||||
else None,
|
||||
)
|
||||
|
||||
completion_tokens = response_tokens or completion_response["usageMetadata"].get("candidatesTokenCount", 0)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio, importlib, os, uuid
|
||||
import json
|
||||
import math
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
|
|
@ -6098,6 +6099,34 @@ def test_vertex_ai_usage_metadata_video_tokens_with_caching():
|
|||
assert result.prompt_tokens_details.audio_tokens == 200
|
||||
|
||||
|
||||
def test_gemini_cached_audio_tokens_billed_at_audio_cache_read_rate():
|
||||
usage: Final = VertexGeminiConfig().calculate_usage(
|
||||
completion_response={
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 10000,
|
||||
"candidatesTokenCount": 10,
|
||||
"totalTokenCount": 10010,
|
||||
"cachedContentTokenCount": 4000,
|
||||
"promptTokensDetails": [
|
||||
{"modality": "TEXT", "tokenCount": 6000},
|
||||
{"modality": "AUDIO", "tokenCount": 4000},
|
||||
],
|
||||
"cacheTokensDetails": [{"modality": "AUDIO", "tokenCount": 4000}],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
model_info: Final = litellm.get_model_info(model="gemini-2.5-flash", custom_llm_provider="gemini")
|
||||
input_rate: Final = model_info.get("input_cost_per_token")
|
||||
audio_cache_read_rate: Final = model_info.get("cache_read_input_audio_token_cost")
|
||||
assert input_rate is not None and audio_cache_read_rate is not None
|
||||
assert audio_cache_read_rate != model_info.get("cache_read_input_token_cost")
|
||||
prompt_cost, _ = litellm.cost_per_token(model="gemini-2.5-flash", custom_llm_provider="gemini", usage_object=usage)
|
||||
assert math.isclose(prompt_cost, 6000 * input_rate + 4000 * audio_cache_read_rate)
|
||||
assert usage.prompt_tokens_details is not None
|
||||
assert usage.prompt_tokens_details.audio_tokens == 4000
|
||||
|
||||
|
||||
def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt():
|
||||
"""Test promptTokensDetails with DOCUMENT modality for PDF inputs.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue