mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(vertex): pin gemini-embedding-2 pricing for embedContent cost tests
The regression tests for Gemini Embedding 2 embedContent usageMetadata called generic_cost_per_token against litellm.model_cost, which is refetched from main's model_prices_and_context_window.json at import time. When main swaps gemini-embedding-2 to input_cost_per_image_token / per-audio-token / per-video-token, the per-image and per-second code paths on this branch see no matching keys and bill zero, so the tests flip red on staging even though the transformation logic they cover is unchanged. Pin a local pricing schema for gemini-embedding-2 (both the bare name and the vertex_ai/-prefixed alias resolved through custom_llm_provider) in a class-level autouse fixture and reference those constants in the expected-cost math. The tests now verify the cost calculator's mapping from usage details to billable dimensions instead of pinning a vendor rate we do not own. Co-authored-by: Krrish Dholakia <krrish-berri-2@users.noreply.github.com>
This commit is contained in:
parent
252c71c0b2
commit
6711a3a438
1 changed files with 45 additions and 4 deletions
|
|
@ -10,6 +10,7 @@ Covers:
|
|||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
|
||||
_build_part_for_input,
|
||||
|
|
@ -21,6 +22,7 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation
|
|||
)
|
||||
from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
|
||||
|
|
@ -299,14 +301,50 @@ class TestProcessResponse:
|
|||
)
|
||||
|
||||
|
||||
_TEXT_TOKEN_RATE = 2e-07
|
||||
_IMAGE_UNIT_RATE = 0.00012
|
||||
_AUDIO_PER_SECOND_RATE = 0.00016
|
||||
_VIDEO_PER_SECOND_RATE = 0.00079
|
||||
_TEST_MODEL_PRICING = {
|
||||
"input_cost_per_audio_per_second": _AUDIO_PER_SECOND_RATE,
|
||||
"input_cost_per_image": _IMAGE_UNIT_RATE,
|
||||
"input_cost_per_token": _TEXT_TOKEN_RATE,
|
||||
"input_cost_per_video_per_second": _VIDEO_PER_SECOND_RATE,
|
||||
"litellm_provider": "vertex_ai-embedding-models",
|
||||
"max_input_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0,
|
||||
"output_vector_size": 3072,
|
||||
"supports_multimodal": True,
|
||||
"uses_embed_content": True,
|
||||
}
|
||||
|
||||
|
||||
class TestProcessEmbedContentResponseUsage:
|
||||
"""Gemini Embedding 2 embedContent usageMetadata must drive spend.
|
||||
|
||||
Regression for multimodal calls recording prompt_tokens=0 / spend=$0.
|
||||
|
||||
The tests pin a local pricing schema for MODEL so the assertions verify the
|
||||
cost calculator's own mapping from usage details to billable dimensions,
|
||||
without depending on whatever pricing keys are currently published in
|
||||
main's model_prices_and_context_window.json (which is refetched at import
|
||||
time and can drop the per-image / per-second keys these code paths read).
|
||||
"""
|
||||
|
||||
MODEL = "gemini-embedding-2"
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _pin_model_pricing(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, self.MODEL, dict(_TEST_MODEL_PRICING))
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"vertex_ai/{self.MODEL}",
|
||||
dict(_TEST_MODEL_PRICING),
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
def test_multimodal_image_preserves_usage_metadata(self):
|
||||
response_json = {
|
||||
"embedding": {"values": [0.1, 0.2, 0.3]},
|
||||
|
|
@ -430,7 +468,7 @@ class TestProcessEmbedContentResponseUsage:
|
|||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
assert prompt_cost == pytest.approx(0.00012)
|
||||
assert prompt_cost == pytest.approx(_IMAGE_UNIT_RATE)
|
||||
|
||||
def test_file_reference_non_image_not_counted_as_image(self):
|
||||
"""A files/... ref resolving to a non-image mime must not be image-counted."""
|
||||
|
|
@ -465,7 +503,7 @@ class TestProcessEmbedContentResponseUsage:
|
|||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
assert prompt_cost == pytest.approx(2.0 * 0.00016)
|
||||
assert prompt_cost == pytest.approx(2.0 * _AUDIO_PER_SECOND_RATE)
|
||||
|
||||
def test_video_plus_audio_does_not_double_bill_text(self):
|
||||
"""Video+audio responses must not get video tokens reassigned to text."""
|
||||
|
|
@ -499,5 +537,8 @@ class TestProcessEmbedContentResponseUsage:
|
|||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
# 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4
|
||||
assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016)
|
||||
assert prompt_cost == pytest.approx(
|
||||
1 * _TEXT_TOKEN_RATE
|
||||
+ 2 * _VIDEO_PER_SECOND_RATE
|
||||
+ 2 * _AUDIO_PER_SECOND_RATE
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue