mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(cost): avoid double billing Gemini embedding modalities
This commit is contained in:
parent
d1fc231545
commit
c698972020
7 changed files with 171 additions and 89 deletions
|
|
@ -707,6 +707,24 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
|
|||
return 0.0
|
||||
|
||||
|
||||
def _calculate_modality_input_cost(
|
||||
model_info: ModelInfo,
|
||||
tokens: int,
|
||||
token_cost_key: str,
|
||||
units: float,
|
||||
unit_cost_key: str,
|
||||
token_fallback_key: str | None = None,
|
||||
cached_tokens: int = 0,
|
||||
) -> float:
|
||||
token_rate: Final = _get_cost_per_unit(model_info, token_cost_key, None)
|
||||
if token_rate is not None and tokens + cached_tokens > 0:
|
||||
return tokens * token_rate
|
||||
unit_rate: Final = _get_cost_per_unit(model_info, unit_cost_key, None)
|
||||
if unit_rate is not None and units > 0:
|
||||
return units * unit_rate
|
||||
return calculate_cost_component(model_info, token_fallback_key or token_cost_key, tokens)
|
||||
|
||||
|
||||
def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
|
||||
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
|
||||
cost_per_unit: Final = model_info.get(cost_key)
|
||||
|
|
@ -955,26 +973,30 @@ def _calculate_input_cost(
|
|||
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
|
||||
)
|
||||
|
||||
### AUDIO COST
|
||||
if prompt_tokens_details["audio_tokens"]:
|
||||
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
|
||||
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
|
||||
|
||||
### IMAGE TOKEN COST
|
||||
if prompt_tokens_details["image_tokens"]:
|
||||
# For image token costs:
|
||||
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
|
||||
image_token_cost_key = "input_cost_per_image_token"
|
||||
if model_info.get(image_token_cost_key) is None:
|
||||
image_token_cost_key = "input_cost_per_token"
|
||||
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
|
||||
|
||||
### VIDEO TOKEN COST
|
||||
if prompt_tokens_details["video_tokens"]:
|
||||
video_token_cost_key = "input_cost_per_video_token"
|
||||
if model_info.get(video_token_cost_key) is None:
|
||||
video_token_cost_key = "input_cost_per_token"
|
||||
prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
|
||||
prompt_cost += _calculate_modality_input_cost(
|
||||
model_info,
|
||||
prompt_tokens_details["audio_tokens"],
|
||||
_get_service_tier_cost_key("input_cost_per_audio_token", service_tier),
|
||||
prompt_tokens_details["audio_length_seconds"],
|
||||
"input_cost_per_audio_per_second",
|
||||
cached_tokens=cache_hit_audio_tokens,
|
||||
)
|
||||
prompt_cost += _calculate_modality_input_cost(
|
||||
model_info,
|
||||
prompt_tokens_details["image_tokens"],
|
||||
"input_cost_per_image_token",
|
||||
prompt_tokens_details["image_count"],
|
||||
"input_cost_per_image",
|
||||
token_fallback_key="input_cost_per_token",
|
||||
)
|
||||
prompt_cost += _calculate_modality_input_cost(
|
||||
model_info,
|
||||
prompt_tokens_details["video_tokens"],
|
||||
"input_cost_per_video_token",
|
||||
prompt_tokens_details["video_length_seconds"],
|
||||
"input_cost_per_video_per_second",
|
||||
token_fallback_key="input_cost_per_token",
|
||||
)
|
||||
|
||||
### CACHE WRITING COST - Now uses tiered pricing
|
||||
if (
|
||||
|
|
@ -996,28 +1018,6 @@ def _calculate_input_cost(
|
|||
prompt_tokens_details["character_count"],
|
||||
)
|
||||
|
||||
### IMAGE COUNT COST
|
||||
if prompt_tokens_details["image_count"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
|
||||
)
|
||||
|
||||
### VIDEO LENGTH COST
|
||||
if prompt_tokens_details["video_length_seconds"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info,
|
||||
"input_cost_per_video_per_second",
|
||||
prompt_tokens_details["video_length_seconds"],
|
||||
)
|
||||
|
||||
### AUDIO LENGTH COST
|
||||
if prompt_tokens_details["audio_length_seconds"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info,
|
||||
"input_cost_per_audio_per_second",
|
||||
prompt_tokens_details["audio_length_seconds"],
|
||||
)
|
||||
|
||||
if prompt_tokens_details["query_count"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info, "input_cost_per_query", prompt_tokens_details["query_count"]
|
||||
|
|
|
|||
|
|
@ -374,25 +374,21 @@ def _usage_from_embed_content_response(
|
|||
details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
|
||||
text_tokens: Final = _tokens_for_modality(details, "TEXT")
|
||||
audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
|
||||
image_tokens: Final = _tokens_for_modality(details, "IMAGE")
|
||||
video_tokens: Final = _tokens_for_modality(details, "VIDEO")
|
||||
image_count: Final = _count_input_images(input, resolved_files)
|
||||
|
||||
video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
|
||||
audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
|
||||
|
||||
# generic_cost_per_token rewrites text_tokens to the full prompt minus
|
||||
# other modalities when both text_tokens and image_count are zero. For
|
||||
# video, that misallocates video tokens to text; a 1-token floor sidesteps
|
||||
# the rewrite and keeps billing on input_cost_per_video_per_second.
|
||||
needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
|
||||
resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
|
||||
|
||||
return Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
total_tokens=total_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=resolved_text_tokens,
|
||||
text_tokens=text_tokens,
|
||||
audio_tokens=audio_tokens,
|
||||
image_tokens=image_tokens,
|
||||
video_tokens=video_tokens,
|
||||
image_count=image_count,
|
||||
video_length_seconds=video_length_seconds,
|
||||
audio_length_seconds=audio_length_seconds,
|
||||
|
|
|
|||
|
|
@ -25602,9 +25602,12 @@
|
|||
},
|
||||
"gemini-embedding-2-preview": {
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai-embedding-models",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25622,6 +25625,7 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai-embedding-models",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25634,9 +25638,12 @@
|
|||
},
|
||||
"vertex_ai/gemini-embedding-2-preview": {
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25655,6 +25662,7 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25694,9 +25702,12 @@
|
|||
"gemini/gemini-embedding-2-preview": {
|
||||
"deprecation_date": "2026-08-10",
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25710,9 +25721,12 @@
|
|||
},
|
||||
"gemini/gemini-embedding-2": {
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
|
|||
|
|
@ -25602,9 +25602,12 @@
|
|||
},
|
||||
"gemini-embedding-2-preview": {
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai-embedding-models",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25622,6 +25625,7 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai-embedding-models",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25634,9 +25638,12 @@
|
|||
},
|
||||
"vertex_ai/gemini-embedding-2-preview": {
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25655,6 +25662,7 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "vertex_ai",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25694,9 +25702,12 @@
|
|||
"gemini/gemini-embedding-2-preview": {
|
||||
"deprecation_date": "2026-08-10",
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
@ -25710,9 +25721,12 @@
|
|||
},
|
||||
"gemini/gemini-embedding-2": {
|
||||
"input_cost_per_audio_per_second": 0.00016,
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
"input_cost_per_image": 0.00012,
|
||||
"input_cost_per_image_token": 4.5e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_video_per_second": 0.00079,
|
||||
"input_cost_per_video_token": 1.2e-05,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
|
|
|
|||
|
|
@ -375,6 +375,10 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_video_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_dbu_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
|
@ -5334,3 +5335,44 @@ def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates(
|
|||
|
||||
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider)
|
||||
assert prompt_cost == pytest.approx(expected_prompt_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens,cached_audio,token_rate,unit_rate,expected", [
|
||||
(10, 0, 0.1, 7.0, 3.0),
|
||||
(10, 0, 0.0, 7.0, 0.0),
|
||||
(10, 0, None, 7.0, 42.0),
|
||||
(0, 0, 0.1, 7.0, 42.0),
|
||||
(0, 10, 0.1, 7.0, 28.0),
|
||||
(10, 0, None, None, 0.2),
|
||||
(10, 0, None, 0.0, 0.0),
|
||||
])
|
||||
def test_modality_tokens_and_physical_units_are_alternative_prices(
|
||||
tokens: int, cached_audio: int, token_rate: float | None,
|
||||
unit_rate: float | None, expected: float,
|
||||
) -> None:
|
||||
info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 0.01,
|
||||
"input_cost_per_audio_token": token_rate,
|
||||
"input_cost_per_image_token": token_rate,
|
||||
"input_cost_per_video_token": token_rate,
|
||||
"input_cost_per_audio_per_second": unit_rate,
|
||||
"input_cost_per_image": unit_rate,
|
||||
"input_cost_per_video_per_second": unit_rate,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
}
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=3 * tokens + cached_audio,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=0, audio_tokens=tokens + cached_audio,
|
||||
image_tokens=tokens, video_tokens=tokens,
|
||||
cached_tokens=cached_audio,
|
||||
cached_tokens_details={"audio_tokens": cached_audio},
|
||||
image_count=2, audio_length_seconds=2.0,
|
||||
video_length_seconds=2.0,
|
||||
),
|
||||
)
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
"modality-unit-test", usage, "vertex_ai", model_info=info,
|
||||
)
|
||||
assert prompt_cost == pytest.approx(expected)
|
||||
assert completion_cost == 0
|
||||
|
|
|
|||
|
|
@ -299,15 +299,25 @@ class TestProcessResponse:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("model,provider", [
|
||||
("gemini-embedding-2", "vertex_ai"),
|
||||
("gemini-embedding-2-preview", "vertex_ai"),
|
||||
("vertex_ai/gemini-embedding-2", "vertex_ai"),
|
||||
("vertex_ai/gemini-embedding-2-preview", "vertex_ai"),
|
||||
("gemini-embedding-2", "gemini"),
|
||||
("gemini-embedding-2-preview", "gemini"),
|
||||
])
|
||||
class TestProcessEmbedContentResponseUsage:
|
||||
"""Gemini Embedding 2 embedContent usageMetadata must drive spend.
|
||||
|
||||
Regression for multimodal calls recording prompt_tokens=0 / spend=$0.
|
||||
"""
|
||||
|
||||
MODEL = "gemini-embedding-2"
|
||||
|
||||
def test_multimodal_image_preserves_usage_metadata(self):
|
||||
@pytest.mark.parametrize("inputs,image_count", [(IMAGE_DATA_URI, 1), ([], 0)])
|
||||
def test_multimodal_image_preserves_usage_metadata(
|
||||
self, model: str, provider: str, inputs: str | list[str], image_count: int,
|
||||
) -> None:
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1, 0.2, 0.3]},
|
||||
"usageMetadata": {
|
||||
|
|
@ -317,23 +327,24 @@ class TestProcessEmbedContentResponseUsage:
|
|||
},
|
||||
}
|
||||
result = process_embed_content_response(
|
||||
input=[IMAGE_DATA_URI],
|
||||
input=inputs,
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
)
|
||||
assert result.usage.prompt_tokens == 258
|
||||
assert result.usage.total_tokens == 258
|
||||
assert result.usage.prompt_tokens_details.image_count == 1
|
||||
assert result.usage.prompt_tokens_details.image_count == image_count
|
||||
assert result.usage.prompt_tokens_details.image_tokens == 258
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
assert prompt_cost > 0
|
||||
assert prompt_cost == pytest.approx(258 * 4.5e-7)
|
||||
|
||||
def test_text_modality_detail_populated(self):
|
||||
def test_text_modality_detail_populated(self, model: str, provider: str) -> None:
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1, 0.2]},
|
||||
"usageMetadata": {
|
||||
|
|
@ -345,20 +356,20 @@ class TestProcessEmbedContentResponseUsage:
|
|||
result = process_embed_content_response(
|
||||
input="a short caption",
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
)
|
||||
assert result.usage.prompt_tokens == 12
|
||||
assert result.usage.prompt_tokens_details.text_tokens == 12
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
assert prompt_cost > 0
|
||||
assert prompt_cost == pytest.approx(12 * 2e-7)
|
||||
|
||||
def test_video_modality_derives_seconds_and_text_floor(self):
|
||||
def test_video_modality_preserves_tokens_without_text(self, model: str, provider: str) -> None:
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1]},
|
||||
"usageMetadata": {
|
||||
|
|
@ -370,38 +381,38 @@ class TestProcessEmbedContentResponseUsage:
|
|||
result = process_embed_content_response(
|
||||
input=["gs://bucket/clip.mp4"],
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
)
|
||||
assert result.usage.prompt_tokens == 516
|
||||
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
|
||||
2.0
|
||||
)
|
||||
assert result.usage.prompt_tokens_details.text_tokens == 1
|
||||
assert result.usage.prompt_tokens_details.video_tokens == 516
|
||||
assert result.usage.prompt_tokens_details.text_tokens == 0
|
||||
|
||||
def test_missing_usage_metadata_does_not_estimate_from_base64(self):
|
||||
def test_missing_usage_metadata_does_not_estimate_from_base64(self, model: str, provider: str) -> None:
|
||||
response_json = {"embedding": {"values": [0.1, 0.2]}}
|
||||
result = process_embed_content_response(
|
||||
input=[IMAGE_DATA_URI],
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
)
|
||||
assert result.usage.prompt_tokens == 0
|
||||
assert result.usage.total_tokens == 0
|
||||
|
||||
def test_missing_usage_metadata_text_falls_back_to_token_counter(self):
|
||||
def test_missing_usage_metadata_text_falls_back_to_token_counter(self, model: str, provider: str) -> None:
|
||||
response_json = {"embedding": {"values": [0.1, 0.2]}}
|
||||
result = process_embed_content_response(
|
||||
input="hello world this is plain text",
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
)
|
||||
assert result.usage.prompt_tokens > 0
|
||||
|
||||
def test_file_reference_image_billed_per_image_not_text(self):
|
||||
"""files/... image refs must bill per-image, not at the text token rate."""
|
||||
def test_file_reference_image_billed_as_image_tokens(self, model: str, provider: str) -> None:
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1, 0.2, 0.3]},
|
||||
"usageMetadata": {
|
||||
|
|
@ -413,7 +424,7 @@ class TestProcessEmbedContentResponseUsage:
|
|||
result = process_embed_content_response(
|
||||
input=["files/img123"],
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
resolved_files={
|
||||
"files/img123": {
|
||||
|
|
@ -426,13 +437,14 @@ class TestProcessEmbedContentResponseUsage:
|
|||
assert result.usage.prompt_tokens_details.text_tokens == 0
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
assert prompt_cost == pytest.approx(0.00012)
|
||||
assert result.usage.prompt_tokens_details.image_tokens == 258
|
||||
assert prompt_cost == pytest.approx(258 * 4.5e-7)
|
||||
|
||||
def test_file_reference_non_image_not_counted_as_image(self):
|
||||
def test_file_reference_non_image_not_counted_as_image(self, model: str, provider: str) -> None:
|
||||
"""A files/... ref resolving to a non-image mime must not be image-counted."""
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1, 0.2]},
|
||||
|
|
@ -445,7 +457,7 @@ class TestProcessEmbedContentResponseUsage:
|
|||
result = process_embed_content_response(
|
||||
input=["files/clip1"],
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
resolved_files={
|
||||
"files/clip1": {
|
||||
|
|
@ -461,13 +473,13 @@ class TestProcessEmbedContentResponseUsage:
|
|||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
assert prompt_cost == pytest.approx(2.0 * 0.00016)
|
||||
assert prompt_cost == pytest.approx(64 * 6.5e-6)
|
||||
|
||||
def test_video_plus_audio_does_not_double_bill_text(self):
|
||||
def test_video_plus_audio_does_not_double_bill_text(self, model: str, provider: str) -> None:
|
||||
"""Video+audio responses must not get video tokens reassigned to text."""
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1]},
|
||||
|
|
@ -483,10 +495,11 @@ class TestProcessEmbedContentResponseUsage:
|
|||
result = process_embed_content_response(
|
||||
input=["gs://bucket/clip.mp4"],
|
||||
model_response=EmbeddingResponse(),
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
response_json=response_json,
|
||||
)
|
||||
assert result.usage.prompt_tokens_details.text_tokens == 1
|
||||
assert result.usage.prompt_tokens_details.video_tokens == 516
|
||||
assert result.usage.prompt_tokens_details.text_tokens == 0
|
||||
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
|
||||
2.0
|
||||
)
|
||||
|
|
@ -495,9 +508,8 @@ class TestProcessEmbedContentResponseUsage:
|
|||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model=self.MODEL,
|
||||
model=model,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
# 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4
|
||||
assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016)
|
||||
assert prompt_cost == pytest.approx(516 * 12e-6 + 64 * 6.5e-6)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue