From 96684a257066ef21e8b2a74e20fa4e353e0ca0c8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 18:17:29 -0700 Subject: [PATCH] fix(vertex_ai): bill per-character text embeddings on Google's billableCharacterCount (#47460) Vertex text embedding predict responses report metadata.billableCharacterCount, the unit Google bills text-embedding-005 and text-multilingual-embedding-002 on. Carry it into usage.prompt_tokens_details.character_count and, when the model carries input_cost_per_character, bill that count instead of the token count so the prompt is charged under one unit. Per-token embeddings (gemini-embedding-001) are unchanged. Backport of #47460 to stable/1.104.x Cherry-picked from d29200fa0511ae50f41b692de5922659a1f176c0 (main) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 8 ++++- .../vertex_embeddings/transformation.py | 34 +++++++++++++++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 25 ++++++++++++++ .../vertex_ai/vertex_embeddings/__init__.py | 0 .../test_embedding_handler.py | 30 ++++++++++++++++ 5 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 tests/unit/llms/vertex_ai/vertex_embeddings/__init__.py create mode 100644 tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e65531121a6..35ebf68e7d1 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1094,8 +1094,14 @@ def _calculate_input_cost( ) -> float: """ Calculates the input cost for a given model, prompt tokens, and completion tokens. + + A model priced per character (`input_cost_per_character`) bills the reported character count + in place of its text tokens, so the same prompt is never charged under both units. """ - prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost + bills_characters: Final = ( + prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None + ) + prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index e4bbdd1bd0d..e4f7c513eb9 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,14 +1,34 @@ import types from typing import Final, Literal -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter +from typing_extensions import ReadOnly, TypedDict from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels -from litellm.types.utils import EmbeddingResponse, Usage +from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage from .types import * +class _PredictMetadata(TypedDict, total=False): + billableCharacterCount: ReadOnly[int] + + +class _PredictResponseMetadata(TypedDict, total=False): + metadata: ReadOnly[_PredictMetadata] + + +_PREDICT_RESPONSE_METADATA: Final = TypeAdapter(_PredictResponseMetadata) + + +def billable_character_count(response: _PredictResponseMetadata) -> int | None: + """ + `metadata.billableCharacterCount` of a Vertex text embedding predict response, the count Google + bills per-character embedding models (text-embedding-005, text-multilingual-embedding-002) on. + """ + return response.get("metadata", {}).get("billableCharacterCount") + + class VertexAITextEmbeddingConfig(BaseModel): """ Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#TextEmbeddingInput @@ -248,7 +268,15 @@ class VertexAITextEmbeddingConfig(BaseModel): model_response.object = "list" model_response.data = embedding_response model_response.model = model - usage: Final = Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens) + billable_characters: Final = billable_character_count(_PREDICT_RESPONSE_METADATA.validate_python(response)) + usage: Final = Usage( + prompt_tokens=input_tokens, + completion_tokens=0, + total_tokens=input_tokens, + prompt_tokens_details=( + None if billable_characters is None else PromptTokensDetailsWrapper(character_count=billable_characters) + ), + ) setattr(model_response, "usage", usage) return model_response diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 2b5dd50fdec..38bea4b2a76 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -99,6 +99,31 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re assert completion_cost == pytest.approx(380 * 9.7e-7) +@pytest.mark.parametrize( + ("model_info", "expected_prompt_cost"), + [ + ({"input_cost_per_token": 1e-7, "input_cost_per_character": 2.5e-8, "output_cost_per_token": 0.0}, 35 * 2.5e-8), + ({"input_cost_per_token": 1.5e-7, "output_cost_per_token": 0.0}, 11 * 1.5e-7), + ], + ids=["character_priced_model_bills_characters_instead_of_tokens", "token_priced_model_ignores_the_character_count"], +) +def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( + model_info: ModelInfo, expected_prompt_cost: float +) -> None: + usage = Usage( + prompt_tokens=11, + completion_tokens=0, + total_tokens=11, + prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0) + + def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: openai_info: ModelInfo = {"input_cost_per_token": 2e-6} fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/__init__.py b/tests/unit/llms/vertex_ai/vertex_embeddings/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py new file mode 100644 index 00000000000..cbc074b4e94 --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py @@ -0,0 +1,30 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.vertex_embeddings.transformation import VertexAITextEmbeddingConfig +from litellm.types.utils import EmbeddingResponse + + +def _gecko_prediction(values: list[float], token_count: int) -> dict[str, object]: + return {"embeddings": {"values": values, "statistics": {"token_count": token_count, "truncated": False}}} + + +@pytest.mark.parametrize( + ("metadata", "character_count"), + [({"metadata": {"billableCharacterCount": 10}}, 10), ({}, None)], + ids=["billable_character_count_reported", "no_metadata"], +) +def test_vertex_embedding_usage_carries_the_billable_character_count_google_prices_on( + metadata: dict[str, object], character_count: int | None +) -> None: + payload: Final = {"predictions": [_gecko_prediction([0.1, 0.2, 0.3], 3), _gecko_prediction([1, 2.0, 3], 5)], **metadata} + + response: Final = VertexAITextEmbeddingConfig().transform_vertex_response_to_openai( + response=payload, model="text-embedding-005", model_response=EmbeddingResponse() + ) + + assert (response.usage.prompt_tokens, getattr(response.usage.prompt_tokens_details, "character_count", None)) == ( + 8, + character_count, + )