From 6711a3a4381ea0dca7a27116a3decc113316dbb5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 18 Sep 2026 18:35:17 +0000 Subject: [PATCH] test(vertex): pin gemini-embedding-2 pricing for embedContent cost tests The regression tests for Gemini Embedding 2 embedContent usageMetadata called generic_cost_per_token against litellm.model_cost, which is refetched from main's model_prices_and_context_window.json at import time. When main swaps gemini-embedding-2 to input_cost_per_image_token / per-audio-token / per-video-token, the per-image and per-second code paths on this branch see no matching keys and bill zero, so the tests flip red on staging even though the transformation logic they cover is unchanged. Pin a local pricing schema for gemini-embedding-2 (both the bare name and the vertex_ai/-prefixed alias resolved through custom_llm_provider) in a class-level autouse fixture and reference those constants in the expected-cost math. The tests now verify the cost calculator's mapping from usage details to billable dimensions instead of pinning a vendor rate we do not own. Co-authored-by: Krrish Dholakia --- ...test_batch_embed_content_transformation.py | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 86b3f0976ab..99477e36652 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -10,6 +10,7 @@ Covers: import pytest +import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, @@ -21,6 +22,7 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation ) from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject from litellm.types.utils import EmbeddingResponse +from litellm.utils import _invalidate_model_cost_lowercase_map IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII" @@ -299,14 +301,50 @@ class TestProcessResponse: ) +_TEXT_TOKEN_RATE = 2e-07 +_IMAGE_UNIT_RATE = 0.00012 +_AUDIO_PER_SECOND_RATE = 0.00016 +_VIDEO_PER_SECOND_RATE = 0.00079 +_TEST_MODEL_PRICING = { + "input_cost_per_audio_per_second": _AUDIO_PER_SECOND_RATE, + "input_cost_per_image": _IMAGE_UNIT_RATE, + "input_cost_per_token": _TEXT_TOKEN_RATE, + "input_cost_per_video_per_second": _VIDEO_PER_SECOND_RATE, + "litellm_provider": "vertex_ai-embedding-models", + "max_input_tokens": 8192, + "max_tokens": 8192, + "mode": "embedding", + "output_cost_per_token": 0, + "output_vector_size": 3072, + "supports_multimodal": True, + "uses_embed_content": True, +} + + class TestProcessEmbedContentResponseUsage: """Gemini Embedding 2 embedContent usageMetadata must drive spend. Regression for multimodal calls recording prompt_tokens=0 / spend=$0. + + The tests pin a local pricing schema for MODEL so the assertions verify the + cost calculator's own mapping from usage details to billable dimensions, + without depending on whatever pricing keys are currently published in + main's model_prices_and_context_window.json (which is refetched at import + time and can drop the per-image / per-second keys these code paths read). """ MODEL = "gemini-embedding-2" + @pytest.fixture(autouse=True) + def _pin_model_pricing(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, self.MODEL, dict(_TEST_MODEL_PRICING)) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{self.MODEL}", + dict(_TEST_MODEL_PRICING), + ) + _invalidate_model_cost_lowercase_map() + def test_multimodal_image_preserves_usage_metadata(self): response_json = { "embedding": {"values": [0.1, 0.2, 0.3]}, @@ -430,7 +468,7 @@ class TestProcessEmbedContentResponseUsage: usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(0.00012) + assert prompt_cost == pytest.approx(_IMAGE_UNIT_RATE) def test_file_reference_non_image_not_counted_as_image(self): """A files/... ref resolving to a non-image mime must not be image-counted.""" @@ -465,7 +503,7 @@ class TestProcessEmbedContentResponseUsage: usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(2.0 * 0.00016) + assert prompt_cost == pytest.approx(2.0 * _AUDIO_PER_SECOND_RATE) def test_video_plus_audio_does_not_double_bill_text(self): """Video+audio responses must not get video tokens reassigned to text.""" @@ -499,5 +537,8 @@ class TestProcessEmbedContentResponseUsage: usage=result.usage, custom_llm_provider="vertex_ai", ) - # 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4 - assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016) + assert prompt_cost == pytest.approx( + 1 * _TEXT_TOKEN_RATE + + 2 * _VIDEO_PER_SECOND_RATE + + 2 * _AUDIO_PER_SECOND_RATE + )