fix(vertex_ai): bill per-character text embeddings on Google's billableCharacterCount (#47460)

Vertex text embedding predict responses report metadata.billableCharacterCount, the
unit Google bills text-embedding-005 and text-multilingual-embedding-002 on. Carry it
into usage.prompt_tokens_details.character_count and, when the model carries
input_cost_per_character, bill that count instead of the token count so the prompt is
charged under one unit. Per-token embeddings (gemini-embedding-001) are unchanged.

Backport of #47460 to stable/1.104.x
Cherry-picked from d29200fa05 (main)

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1094,8 +1094,14 @@ def _calculate_input_cost(
 ) -> float:
     """
     Calculates the input cost for a given model, prompt tokens, and completion tokens.
+
+    A model priced per character (`input_cost_per_character`) bills the reported character count
+    in place of its text tokens, so the same prompt is never charged under both units.
     """
-    prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
+    bills_characters: Final = (
+        prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None
+    )
+    prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
 
     ### CACHE READ COST - Now uses tiered pricing
     cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]

litellm/llms/vertex_ai/vertex_embeddings/transformation.py

@@ -1,14 +1,34 @@
 import types
 from typing import Final, Literal
 
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
-from litellm.types.utils import EmbeddingResponse, Usage
+from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage
 
 from .types import *
 
 
+class _PredictMetadata(TypedDict, total=False):
+    billableCharacterCount: ReadOnly[int]
+
+
+class _PredictResponseMetadata(TypedDict, total=False):
+    metadata: ReadOnly[_PredictMetadata]
+
+
+_PREDICT_RESPONSE_METADATA: Final = TypeAdapter(_PredictResponseMetadata)
+
+
+def billable_character_count(response: _PredictResponseMetadata) -> int | None:
+    """
+    `metadata.billableCharacterCount` of a Vertex text embedding predict response, the count Google
+    bills per-character embedding models (text-embedding-005, text-multilingual-embedding-002) on.
+    """
+    return response.get("metadata", {}).get("billableCharacterCount")
+
+
 class VertexAITextEmbeddingConfig(BaseModel):
     """
     Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#TextEmbeddingInput
@@ -248,7 +268,15 @@ class VertexAITextEmbeddingConfig(BaseModel):
         model_response.object = "list"
         model_response.data = embedding_response
         model_response.model = model
-        usage: Final = Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens)
+        billable_characters: Final = billable_character_count(_PREDICT_RESPONSE_METADATA.validate_python(response))
+        usage: Final = Usage(
+            prompt_tokens=input_tokens,
+            completion_tokens=0,
+            total_tokens=input_tokens,
+            prompt_tokens_details=(
+                None if billable_characters is None else PromptTokensDetailsWrapper(character_count=billable_characters)
+            ),
+        )
         setattr(model_response, "usage", usage)
         return model_response
 

tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py

@@ -99,6 +99,31 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re
     assert completion_cost == pytest.approx(380 * 9.7e-7)
 
 
+@pytest.mark.parametrize(
+    ("model_info", "expected_prompt_cost"),
+    [
+        ({"input_cost_per_token": 1e-7, "input_cost_per_character": 2.5e-8, "output_cost_per_token": 0.0}, 35 * 2.5e-8),
+        ({"input_cost_per_token": 1.5e-7, "output_cost_per_token": 0.0}, 11 * 1.5e-7),
+    ],
+    ids=["character_priced_model_bills_characters_instead_of_tokens", "token_priced_model_ignores_the_character_count"],
+)
+def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit(
+    model_info: ModelInfo, expected_prompt_cost: float
+) -> None:
+    usage = Usage(
+        prompt_tokens=11,
+        completion_tokens=0,
+        total_tokens=11,
+        prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35),
+    )
+
+    prompt_cost, completion_cost = generic_cost_per_token(
+        model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info
+    )
+
+    assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0)
+
+
 def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None:
     openai_info: ModelInfo = {"input_cost_per_token": 2e-6}
     fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6}

tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py

@@ -0,0 +1,30 @@
+from typing import Final
+
+import pytest
+
+from litellm.llms.vertex_ai.vertex_embeddings.transformation import VertexAITextEmbeddingConfig
+from litellm.types.utils import EmbeddingResponse
+
+
+def _gecko_prediction(values: list[float], token_count: int) -> dict[str, object]:
+    return {"embeddings": {"values": values, "statistics": {"token_count": token_count, "truncated": False}}}
+
+
+@pytest.mark.parametrize(
+    ("metadata", "character_count"),
+    [({"metadata": {"billableCharacterCount": 10}}, 10), ({}, None)],
+    ids=["billable_character_count_reported", "no_metadata"],
+)
+def test_vertex_embedding_usage_carries_the_billable_character_count_google_prices_on(
+    metadata: dict[str, object], character_count: int | None
+) -> None:
+    payload: Final = {"predictions": [_gecko_prediction([0.1, 0.2, 0.3], 3), _gecko_prediction([1, 2.0, 3], 5)], **metadata}
+
+    response: Final = VertexAITextEmbeddingConfig().transform_vertex_response_to_openai(
+        response=payload, model="text-embedding-005", model_response=EmbeddingResponse()
+    )
+
+    assert (response.usage.prompt_tokens, getattr(response.usage.prompt_tokens_details, "character_count", None)) == (
+        8,
+        character_count,
+    )