test: align Gemini embedding pricing assertions

This commit is contained in:
李怡霖 · Yilin Li 2026-09-18 11:56:24 +08:00
parent 20b8a3055d
commit d992d24679
11 changed files with 177 additions and 494 deletions

View file

@ -921,16 +921,12 @@ def _calculate_input_cost(
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
### AUDIO COST
if prompt_tokens_details["audio_tokens"] and not (
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
):
if prompt_tokens_details["audio_tokens"]:
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"] and not (
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
):
if prompt_tokens_details["image_tokens"]:
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
@ -939,9 +935,7 @@ def _calculate_input_cost(
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"] and not (
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
):
if prompt_tokens_details["video_tokens"]:
video_token_cost_key = "input_cost_per_video_token"
if model_info.get(video_token_cost_key) is None:
video_token_cost_key = "input_cost_per_token"

View file

@ -298,6 +298,8 @@ def transform_openai_input_gemini_embed_content(
_IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"})
_VIDEO_TOKENS_PER_SECOND: Final = 258.0
_AUDIO_TOKENS_PER_SECOND: Final = 32.0
_usage_metadata_adapter: Final = TypeAdapter(UsageMetadata)
@ -337,12 +339,11 @@ def _is_image_element(
return False
def _is_image_only_input(
def _count_input_images(
input: GeminiEmbeddingInput,
resolved_files: Mapping[str, Mapping[str, str]],
) -> bool:
elements: Final = _flatten_input(input)
return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements)
) -> int:
return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files))
def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int:
@ -371,29 +372,30 @@ def _usage_from_embed_content_response(
total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens
details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
if not details:
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0,
image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0,
),
)
text_tokens: Final = _tokens_for_modality(details, "TEXT")
audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
image_tokens: Final = _tokens_for_modality(details, "IMAGE")
video_tokens: Final = _tokens_for_modality(details, "VIDEO")
image_count: Final = _count_input_images(input, resolved_files)
video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
# generic_cost_per_token rewrites text_tokens to the full prompt minus
# other modalities when both text_tokens and image_count are zero. For
# video, that misallocates video tokens to text; a 1-token floor sidesteps
# the rewrite and keeps billing on input_cost_per_video_per_second.
needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=text_tokens,
text_tokens=resolved_text_tokens,
audio_tokens=audio_tokens,
image_tokens=image_tokens,
video_tokens=video_tokens,
image_count=image_count,
video_length_seconds=video_length_seconds,
audio_length_seconds=audio_length_seconds,
),
)
@ -413,7 +415,8 @@ def process_embed_content_response(
model_response: EmbeddingResponse to populate
model: Model name
response_json: Raw JSON response from embedContent endpoint
resolved_files: Mapping of file references to resolved metadata
resolved_files: Mapping of file references (files/abc) to {mime_type, uri},
used to bill resolved image references at the per-image rate
Returns:
EmbeddingResponse with single embedding

View file

@ -25380,11 +25380,10 @@
"uses_embed_content": true
},
"gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25411,11 +25410,10 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25470,11 +25468,10 @@
"tpm": 10000000
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -65274,4 +65271,4 @@
"supports_response_schema": true,
"supports_vision": false
}
}
}

View file

@ -30,8 +30,8 @@ This pre-call check is responsible only for the routing decision: it reads the e
the matching deployment.
Safe to enable globally:
- Only pins requests carrying encoded markers in ``input`` or assistant ``reasoning_items``.
- No effect on embedding models or first-time deployment selection.
- Only activates when encoded markers appear in the request ``input``.
- No effect on embedding models, chat completions, or first-time requests.
- No quota reduction -- first requests are fully load balanced.
- No cache required.
"""
@ -71,8 +71,8 @@ class _SupportsActiveCooldowns(Protocol):
class EncryptedContentAffinityCheck(CustomLogger):
"""
Routes follow-up Responses API and bridged chat requests to the deployment
that produced the encrypted output items they reference.
Routes follow-up Responses API requests to the deployment that produced
the encrypted output items they reference.
The ``model_id`` is decoded directly from the litellm-encoded item IDs –
no caching or TTL management needed.
@ -184,27 +184,6 @@ class EncryptedContentAffinityCheck(CustomLogger):
None,
)
@staticmethod
def _extract_model_id_from_messages(
messages: list[AllMessageValues] | None,
) -> str | None:
if messages is None:
return None
return next(
(
model_id
for message in messages
if message.get("role") == "assistant"
if (
model_id := EncryptedContentAffinityCheck._extract_model_id_from_input(
message.get("reasoning_items")
)
)
is not None
),
None,
)
@staticmethod
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
for deployment in healthy_deployments:
@ -218,10 +197,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
@staticmethod
def _request_team_id(request_kwargs: Mapping[str, object]) -> str | None:
containers: Final = (
request_kwargs.get("metadata"),
request_kwargs.get("litellm_metadata"),
)
containers: Final = (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata"))
team_ids: Final = (c.get("user_api_key_team_id") for c in containers if isinstance(c, Mapping))
return next((tid for tid in team_ids if isinstance(tid, str)), None)
@ -316,7 +292,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
remaining cooldown window) so OpenAI-compatible clients back off and
retry after the deployment is eligible again.
"""
routing_kwargs: Final = request_kwargs if request_kwargs is not None else {}
request_kwargs = request_kwargs or {}
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
if not self._is_enabled_for_model_group(model):
return typed_healthy_deployments
@ -328,20 +304,14 @@ class EncryptedContentAffinityCheck(CustomLogger):
# completions / embeddings, which breaks tag-based routing because
# _get_metadata_variable_name_from_kwargs would pick "litellm_metadata"
# over "metadata" where tags are actually stored.
if "litellm_metadata" in routing_kwargs or messages is not None:
metadata_key: Final = "litellm_metadata" if "litellm_metadata" in routing_kwargs else "metadata"
routing_kwargs[metadata_key] = {
**(routing_kwargs.get(metadata_key) or {}),
"encrypted_content_affinity_enabled": True,
}
if "litellm_metadata" in request_kwargs:
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True
request_input: Final = routing_kwargs.get("input")
anthropic_messages: Final = messages or routing_kwargs.get("messages")
model_id: Final = (
self._extract_model_id_from_input(request_input)
or self._extract_model_id_from_messages(messages)
or self._extract_model_id_from_anthropic_messages(anthropic_messages)
)
request_input: Final = request_kwargs.get("input")
anthropic_messages: Final = messages or request_kwargs.get("messages")
model_id: Final = self._extract_model_id_from_input(
request_input
) or self._extract_model_id_from_anthropic_messages(anthropic_messages)
if not model_id:
return typed_healthy_deployments
@ -359,7 +329,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
"EncryptedContentAffinityCheck: pinning -> deployment=%s",
model_id,
)
routing_kwargs["_encrypted_content_affinity_pinned"] = True
request_kwargs["_encrypted_content_affinity_pinned"] = True
return [deployment]
# Follow-up switched model_name (LIT-2531): pin by Azure resource instead.
@ -374,7 +344,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
model_id,
len(boundary_matches),
)
routing_kwargs["_encrypted_content_affinity_pinned"] = True
request_kwargs["_encrypted_content_affinity_pinned"] = True
return boundary_matches
# The origin cannot serve this turn's routed group and no peer shares the boundary, so its
@ -388,7 +358,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
# dispatch rather than returning distinguishable responses. Only a genuine same-group member
# that is currently unavailable falls through to the fail-fast, preserving the cooldown contract.
routed_group_model_ids: Final = (
self._routed_group_candidate_model_ids(routing_kwargs, model) if originating is not None else frozenset()
self._routed_group_candidate_model_ids(request_kwargs, model) if originating is not None else frozenset()
)
if str(model_id) not in routed_group_model_ids:
verbose_router_logger.debug(

View file

@ -25380,11 +25380,10 @@
"uses_embed_content": true
},
"gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25411,11 +25410,10 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25470,11 +25468,10 @@
"tpm": 10000000
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -65274,4 +65271,4 @@
"supports_response_schema": true,
"supports_vision": false
}
}
}

View file

@ -375,10 +375,6 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"input_cost_per_video_token": {
"type": "number",
"minimum": 0
},
"input_dbu_cost_per_token": {
"type": "number",
"minimum": 0

View file

@ -22,11 +22,6 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
pytestmark = pytest.mark.asyncio(loop_scope="session")
def _frozen_cache() -> UserApiKeyCache:
"""The org entries carry a 5s TTL; a frozen clock keeps a slow first call from expiring them mid-test."""
return UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1_000_000.0), redis_cache=None)
def _dead_db() -> MagicMock:
prisma = MagicMock(name="prisma_client")
prisma.db.query_first = AsyncMock(return_value=None)
@ -63,10 +58,9 @@ async def test_join_binds_the_membership_to_the_requested_team(prisma):
data={"user_id": user_id, "team_id": team_b, "litellm_budget_table": {"connect": {"budget_id": f"b-{run}"}}}
)
cache = _frozen_cache()
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None)
refs = AuthObjectRefs(user_id=user_id, team_id=team_a, membership_user_id=user_id, organization_id=org_id)
await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma)
assert cache.in_memory_cache.get_cache(f"org_id:{org_id}") is not None
dead_db = _dead_db()
membership = await get_team_membership(
@ -106,7 +100,7 @@ async def test_join_reads_team_model_aliases_from_the_mapped_column(prisma):
where={"team_id": team_id}, include={"litellm_model_table": True}
)
cache = _frozen_cache()
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None)
refs = AuthObjectRefs(user_id=None, team_id=team_id, membership_user_id=None, organization_id=None)
await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma)
@ -150,7 +144,7 @@ async def test_join_reads_null_nested_lists_the_way_prisma_does(prisma):
where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}}, include={"litellm_budget_table": True}
)
cache = _frozen_cache()
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None)
refs = AuthObjectRefs(user_id=user_id, team_id=team_id, membership_user_id=user_id, organization_id=None)
await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma)

View file

@ -74,36 +74,6 @@ def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, s
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4])
def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None:
model_info: ModelInfo = {
"key": "gemini-embedding-2",
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": 2e-7,
"input_cost_per_audio_token": 6.5e-6,
"input_cost_per_audio_per_second": 0.00016,
"output_cost_per_token": 0.0,
"litellm_provider": "vertex_ai",
"mode": "embedding",
}
usage = Usage(
prompt_tokens=64,
completion_tokens=0,
total_tokens=64,
prompt_tokens_details=PromptTokensDetailsWrapper(
audio_tokens=64,
audio_length_seconds=2,
),
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-embedding-2",
usage=usage,
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(2 * 0.00016)
def test_missing_cache_read_uses_off_peak_input_rate():
from datetime import datetime, timezone
@ -5176,4 +5146,4 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(
assert breakdown.reasoning_cost == pytest.approx(20 * info["output_cost_per_token"])
assert completion_cost == pytest.approx(
30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"]
)
)

View file

@ -10,7 +10,6 @@ Covers:
import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
_build_part_for_input,
@ -23,19 +22,11 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation
from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject
from litellm.types.utils import EmbeddingResponse
IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
GCS_URL = "gs://my-bucket/image.png"
@pytest.fixture(autouse=True)
def _local_model_cost_map(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
class TestIsMultimodalInput:
def test_text_only_string(self):
assert _is_multimodal_input("hello world") is False
@ -333,7 +324,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert result.usage.prompt_tokens == 258
assert result.usage.total_tokens == 258
assert result.usage.prompt_tokens_details.image_tokens == 258
assert result.usage.prompt_tokens_details.image_count == 1
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
@ -367,7 +358,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert prompt_cost > 0
def test_video_modality_preserves_token_count(self):
def test_video_modality_derives_seconds_and_text_floor(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
@ -383,8 +374,10 @@ class TestProcessEmbedContentResponseUsage:
response_json=response_json,
)
assert result.usage.prompt_tokens == 516
assert result.usage.prompt_tokens_details.video_tokens == 516
assert result.usage.prompt_tokens_details.text_tokens == 0
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.text_tokens == 1
def test_missing_usage_metadata_does_not_estimate_from_base64(self):
response_json = {"embedding": {"values": [0.1, 0.2]}}
@ -407,7 +400,8 @@ class TestProcessEmbedContentResponseUsage:
)
assert result.usage.prompt_tokens > 0
def test_file_reference_image_billed_per_image_token_rate(self):
def test_file_reference_image_billed_per_image_not_text(self):
"""files/... image refs must bill per-image, not at the text token rate."""
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
"usageMetadata": {
@ -428,7 +422,7 @@ class TestProcessEmbedContentResponseUsage:
}
},
)
assert result.usage.prompt_tokens_details.image_tokens == 258
assert result.usage.prompt_tokens_details.image_count == 1
assert result.usage.prompt_tokens_details.text_tokens == 0
prompt_cost, _ = generic_cost_per_token(
@ -436,10 +430,10 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(258 * 4.5e-7)
assert prompt_cost == pytest.approx(0.00012)
def test_file_reference_non_image_not_counted_as_image(self):
"""A files/... ref resolving to a non-image mime keeps audio token billing."""
"""A files/... ref resolving to a non-image mime must not be image-counted."""
response_json = {
"embedding": {"values": [0.1, 0.2]},
"usageMetadata": {
@ -460,18 +454,21 @@ class TestProcessEmbedContentResponseUsage:
}
},
)
assert result.usage.prompt_tokens_details.image_count == 0
assert result.usage.prompt_tokens_details.audio_tokens == 64
assert result.usage.prompt_tokens_details.image_tokens == 0
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(
2.0
)
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(64 * 6.5e-6)
assert prompt_cost == pytest.approx(2.0 * 0.00016)
def test_video_plus_audio_does_not_double_bill_text(self):
"""Video and audio responses are billed from their respective token counts."""
"""Video+audio responses must not get video tokens reassigned to text."""
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
@ -489,84 +486,18 @@ class TestProcessEmbedContentResponseUsage:
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.text_tokens == 0
assert result.usage.prompt_tokens_details.video_tokens == 516
assert result.usage.prompt_tokens_details.audio_tokens == 64
assert result.usage.prompt_tokens_details.text_tokens == 1
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(
2.0
)
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6)
def test_image_without_modality_details_uses_image_rate(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 258,
"totalTokenCount": 258,
},
}
result = process_embed_content_response(
input=IMAGE_DATA_URI,
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.image_tokens == 258
assert result.usage.prompt_tokens_details.text_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(258 * 4.5e-7)
def test_mixed_text_and_image_without_modality_details_not_billed_as_image(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 270,
"totalTokenCount": 270,
},
}
result = process_embed_content_response(
input=["a short caption", IMAGE_DATA_URI],
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.image_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(270 * 2e-7)
def test_text_without_modality_details_uses_text_rate(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 12,
"totalTokenCount": 12,
},
}
result = process_embed_content_response(
input="a short caption",
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.text_tokens == 0
assert result.usage.prompt_tokens_details.image_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(12 * 2e-7)
# 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4
assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016)

View file

@ -18,8 +18,9 @@ The mechanism works without any cache and supports two encoding strategies:
import time
from unittest.mock import AsyncMock, patch
import litellm
import pytest
import litellm
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
@ -65,9 +66,7 @@ class TestEncryptedItemIdCodec:
def test_roundtrip(self):
model_id = "deployment-1"
original_item_id = "rs_abc123def456"
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_item_id
)
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id)
assert encoded.startswith("encitem_")
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded)
assert decoded is not None
@ -78,9 +77,7 @@ class TestEncryptedItemIdCodec:
"""Decoding must succeed even if base64 padding (=) was stripped in transit."""
model_id = "gpt-5.1-codex-openai-2"
original_item_id = "rs_0efb96cb222403210069a01d5d52588196a9dc394ffdb89d00"
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_item_id
)
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id)
# Strip any trailing '=' to simulate what happens in transit
stripped = encoded.rstrip("=")
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(stripped)
@ -97,9 +94,7 @@ class TestEncryptedItemIdCodec:
"""item_id values containing ';' must survive the roundtrip."""
model_id = "deployment-1"
original_item_id = "rs_part1;part2;part3"
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_item_id
)
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id)
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded)
assert decoded is not None
assert decoded["item_id"] == original_item_id
@ -115,11 +110,7 @@ class TestUpdateEncryptedContentItemIds:
{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"},
],
}
result = (
ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, model_id
)
)
result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, model_id)
# Plain message item untouched
assert result["output"][0]["id"] == "msg_abc"
# Reasoning item with encrypted_content gets encoded
@ -130,16 +121,8 @@ class TestUpdateEncryptedContentItemIds:
assert decoded["item_id"] == "rs_xyz"
def test_no_op_when_model_id_is_none(self):
response = {
"output": [
{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}
]
}
result = (
ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, None
)
)
response = {"output": [{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}]}
result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, None)
assert result["output"][0]["id"] == "rs_xyz"
@ -148,9 +131,7 @@ class TestEncryptedContentWrapping:
"""Test wrapping encrypted_content with model_id metadata."""
model_id = "deployment-1"
original_content = "gAAAAABpnW_yEYmSNEyOG_original_encrypted_data"
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id)
assert wrapped.startswith("litellm_enc:")
assert wrapped != original_content
@ -167,9 +148,7 @@ class TestEncryptedContentWrapping:
(
model_id,
content,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
plain_content
)
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(plain_content)
assert model_id is None
assert content == plain_content
@ -186,11 +165,7 @@ class TestEncryptedContentWrapping:
},
],
}
result = (
ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, model_id
)
)
result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, model_id)
assert result["output"][0].get("encrypted_content") is None
wrapped = result["output"][1]["encrypted_content"]
assert wrapped.startswith("litellm_enc:")
@ -207,19 +182,13 @@ class TestRestoreEncryptedContentItemIds:
def test_restores_encoded_ids(self):
model_id = "deployment-1"
original_id = "rs_encrypted_item_456"
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_id
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_id)
request_input = [
{"type": "message", "id": "msg_abc123", "role": "assistant"},
{"type": "reasoning", "id": encoded_id, "encrypted_content": "secret"},
]
restored = (
ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
)
)
restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input)
assert restored[0]["id"] == "msg_abc123"
assert restored[1]["id"] == original_id
@ -227,33 +196,21 @@ class TestRestoreEncryptedContentItemIds:
"""Test that wrapped encrypted_content is unwrapped before forwarding."""
model_id = "deployment-1"
original_content = "gAAAAABpnW_yEYmSNEyOG_original"
wrapped_content = (
ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
)
wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id)
request_input = [
{"type": "reasoning", "encrypted_content": wrapped_content},
]
restored = (
ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
)
)
restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input)
assert restored[0]["encrypted_content"] == original_content
def test_no_op_for_plain_string_input(self):
result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
"Hello world"
)
result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input("Hello world")
assert result == "Hello world"
def test_no_op_for_unencoded_ids(self):
request_input = [{"type": "message", "id": "msg_plain"}]
result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
)
result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input)
assert result[0]["id"] == "msg_plain"
@ -280,9 +237,7 @@ async def test_encrypted_content_affinity_tracks_and_routes():
"id": "msg_abc123",
"status": "completed",
"role": "assistant",
"content": [
{"type": "output_text", "text": "Hello!", "annotations": []}
],
"content": [{"type": "output_text", "text": "Hello!", "annotations": []}],
},
{
"type": "reasoning",
@ -475,9 +430,7 @@ async def test_encrypted_content_affinity_bypasses_rpm_limits():
# Extract encoded item ID from the first response output
encoded_item_id = _extract_encoded_item_id(first_response)
assert encoded_item_id.startswith("encitem_"), (
f"Expected encitem_... but got {encoded_item_id!r}"
)
assert encoded_item_id.startswith("encitem_"), f"Expected encitem_... but got {encoded_item_id!r}"
# Follow-up with the encoded item ID — should pin to same deployment
second_response = await router.aresponses(
@ -625,17 +578,13 @@ async def test_encrypted_content_affinity_with_wrapped_content_no_id():
if hasattr(first_item, "encrypted_content")
else first_item.get("encrypted_content")
)
assert wrapped_content.startswith("litellm_enc:"), (
f"Expected wrapped content but got {wrapped_content[:50]}..."
)
assert wrapped_content.startswith("litellm_enc:"), f"Expected wrapped content but got {wrapped_content[:50]}..."
# Verify we can extract model_id from wrapped content
(
extracted_model_id,
_,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
wrapped_content
)
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped_content)
assert extracted_model_id == first_model_id
# Second request: use wrapped encrypted_content WITHOUT an ID (Codex behavior)
@ -661,13 +610,9 @@ def test_encrypted_content_wrapping_preserves_original_content():
This is critical for streaming responses where content must round-trip correctly.
"""
model_id = "test-deployment-1"
original_encrypted_content = (
"gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/"
)
original_encrypted_content = "gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/"
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_encrypted_content, model_id
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_encrypted_content, model_id)
assert wrapped.startswith("litellm_enc:")
assert wrapped != original_encrypted_content
@ -688,9 +633,7 @@ def test_encrypted_content_wrapping_with_multiple_semicolons():
model_id = "deployment-with-semicolons"
original_content = "gAAAAAB;some;content;with;semicolons"
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id)
(
extracted_model_id,
@ -761,9 +704,7 @@ async def test_encrypted_content_affinity_preserves_litellm_metadata_for_respons
request_kwargs=request_kwargs,
)
assert (
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
)
assert request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
assert request_kwargs["litellm_metadata"]["model_info"] == {"id": "dep-1"}
@ -774,9 +715,7 @@ def test_encrypted_content_wrapping_empty_string():
model_id = "test-deployment"
original_content = ""
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id)
assert wrapped.startswith("litellm_enc:")
@ -1129,9 +1068,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance():
"api_key": "fake-azure-resource-key-a",
}
pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key(
pydantic_params
)
pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key(pydantic_params)
plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params)
assert pydantic_key is not None
@ -1158,18 +1095,8 @@ def test_boundary_key_rejects_non_dict_like_inputs():
for bad in (None, [], "not a dict", 42, object()):
assert EncryptedContentAffinityCheck._encryption_boundary_key(bad) is None
assert (
EncryptedContentAffinityCheck._encryption_boundary_key(
{"api_base": "", "api_key": "k"}
)
is None
)
assert (
EncryptedContentAffinityCheck._encryption_boundary_key(
{"api_base": "https://x"}
)
is None
)
assert EncryptedContentAffinityCheck._encryption_boundary_key({"api_base": "", "api_key": "k"}) is None
assert EncryptedContentAffinityCheck._encryption_boundary_key({"api_base": "https://x"}) is None
# ---------------------------------------------------------------------------
@ -1205,12 +1132,8 @@ def _make_router_mock_with_cooldown(
mock_router = MagicMock()
mock_router.get_deployment.return_value = originating
mock_router.cooldown_cache.async_get_active_cooldowns = AsyncMock(
return_value=list(cooldown_entries or [])
)
mock_router.get_candidate_model_ids_for_route.return_value = frozenset(
routed_group_model_ids or []
)
mock_router.cooldown_cache.async_get_active_cooldowns = AsyncMock(return_value=list(cooldown_entries or []))
mock_router.get_candidate_model_ids_for_route.return_value = frozenset(routed_group_model_ids or [])
return mock_router
@ -1245,9 +1168,7 @@ async def test_affinity_raises_service_unavailable_when_origin_cooled_for_non_42
)
check = EncryptedContentAffinityCheck(router=mock_router)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-a-cooled", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-cooled", "rs_test")
healthy_only_b = [
{
"model_info": {"id": "deployment-b"},
@ -1309,9 +1230,7 @@ async def test_affinity_raises_rate_limit_with_retry_after_when_origin_cooled_fo
)
check = EncryptedContentAffinityCheck(router=mock_router)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-a-cooled-429", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-cooled-429", "rs_test")
healthy_only_b = [
{
"model_info": {"id": "deployment-b"},
@ -1356,15 +1275,11 @@ async def test_affinity_raises_service_unavailable_when_origin_filtered_without_
originating = _make_originating_mock("https://account-a.openai.azure.com/", "key-a")
mock_router = _make_router_mock_with_cooldown(
originating,
cooldown_entries=[],
routed_group_model_ids=["deployment-a-filtered", "deployment-b"],
originating, cooldown_entries=[], routed_group_model_ids=["deployment-a-filtered", "deployment-b"]
)
check = EncryptedContentAffinityCheck(router=mock_router)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-a-filtered", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-filtered", "rs_test")
healthy_only_b = [
{
"model_info": {"id": "deployment-b"},
@ -1412,9 +1327,7 @@ async def test_affinity_strips_and_dispatches_when_origin_is_unknown_or_removed(
mock_router.get_deployment.return_value = None
check = EncryptedContentAffinityCheck(router=mock_router)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA-blob", "deployment-removed"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-removed")
routed_pool = [
{
"model_info": {"id": "deployment-b"},
@ -1447,10 +1360,7 @@ async def test_affinity_strips_and_dispatches_when_origin_is_unknown_or_removed(
)
assert result is routed_pool
assert not any(
isinstance(item, dict) and item.get("encrypted_content")
for item in request_kwargs["input"]
)
assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in request_kwargs["input"])
mock_router.get_candidate_model_ids_for_route.assert_not_called()
@ -1476,9 +1386,7 @@ async def test_affinity_does_not_raise_when_boundary_peer_available():
mock_router.get_deployment.return_value = originating
check = EncryptedContentAffinityCheck(router=mock_router)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-a", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a", "rs_test")
peer = {
"model_info": {"id": "deployment-a-peer"},
"litellm_params": {
@ -1522,9 +1430,7 @@ async def test_model_group_affinity_config_enables_encrypted_content_affinity():
},
target_deployment,
]
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-b", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test")
request_kwargs = {
"input": [{"type": "reasoning", "id": encoded_id}],
"litellm_metadata": {},
@ -1568,9 +1474,7 @@ async def test_model_group_affinity_config_does_not_disable_global_encrypted_con
},
target_deployment,
]
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-b", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test")
request_kwargs = {
"input": [{"type": "reasoning", "id": encoded_id}],
"litellm_metadata": {},
@ -1632,15 +1536,9 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen
try:
callbacks = router.optional_callbacks or []
deployment_callback = next(
cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck)
)
encrypted_content_callback = next(
cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck)
)
assert callbacks.index(encrypted_content_callback) < callbacks.index(
deployment_callback
)
deployment_callback = next(cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck))
encrypted_content_callback = next(cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck))
assert callbacks.index(encrypted_content_callback) < callbacks.index(deployment_callback)
assert encrypted_content_callback.enable_global_affinity is False
cache_key = DeploymentAffinityCheck.get_affinity_cache_key(
@ -1652,9 +1550,7 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen
value={"model_id": "deployment-a"},
ttl=60,
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-b", "rs_test"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test")
request_kwargs = {
"input": [
{
@ -1675,13 +1571,11 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen
)
assert after_deployment_affinity == [deployment_a, deployment_b]
after_encrypted_content_affinity = (
await encrypted_content_callback.async_filter_deployments(
model=model_group,
healthy_deployments=after_deployment_affinity,
messages=None,
request_kwargs=request_kwargs,
)
after_encrypted_content_affinity = await encrypted_content_callback.async_filter_deployments(
model=model_group,
healthy_deployments=after_deployment_affinity,
messages=None,
request_kwargs=request_kwargs,
)
assert after_encrypted_content_affinity == [deployment_b]
@ -1700,14 +1594,8 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu
"""
check = EncryptedContentAffinityCheck()
deployments = [
{
"model_info": {"id": "openai-org-a"},
"litellm_params": {"model": "openai/gpt-5.1"},
},
{
"model_info": {"id": "openai-org-b"},
"litellm_params": {"model": "openai/gpt-5.1"},
},
{"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}},
{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}},
]
request_kwargs = {"model": "gpt-5.1"}
@ -1723,23 +1611,14 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu
def _bridge_replayed_anthropic_messages(minted_by: str) -> list:
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA_turn_one", minted_by
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by)
return [
{"role": "user", "content": "Solve the zebra puzzle"},
{
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": "Anthropic minted this one",
"signature": "ErcCCpIBCBEYAipA",
},
{
"type": "redacted_thinking",
"data": f"litellm_encrypted_reasoning:{wrapped}",
},
{"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"},
{"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"},
{
"type": "thinking",
"thinking": "The bridge packed this one",
@ -1766,12 +1645,7 @@ async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_
originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"]
)
check = EncryptedContentAffinityCheck(router=mock_router)
routed_pool = [
{
"model_info": {"id": "openai-org-b"},
"litellm_params": {"model": "openai/gpt-5-nano"},
}
]
routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}]
messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a")
assistant_content = messages[1]["content"]
request_kwargs = {"model": "gpt-5.1"}
@ -1787,26 +1661,16 @@ async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_
assert "_encrypted_content_affinity_pinned" not in request_kwargs
assert messages[1]["content"] is assistant_content
assert assistant_content == [
{
"type": "thinking",
"thinking": "Anthropic minted this one",
"signature": "ErcCCpIBCBEYAipA",
},
{"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"},
{"type": "text", "text": "The zebra owner lives in the green house."},
]
assert all(
block["signature"] for block in assistant_content if block["type"] == "thinking"
)
assert all(block["signature"] for block in assistant_content if block["type"] == "thinking")
class TestStripEncryptedReasoningFromInput:
def test_keeps_summary_and_drops_encrypted_content_and_id(self):
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA-blob", "deployment-a"
)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
"deployment-a", "rs_1"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a")
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a", "rs_1")
request_input = [
{"role": "user", "content": "first turn"},
{
@ -1832,43 +1696,26 @@ class TestStripEncryptedReasoningFromInput:
]
def test_keeps_string_form_summary_when_stripping(self):
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA-blob", "deployment-a"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a")
request_input = [
{
"type": "reasoning",
"encrypted_content": wrapped,
"summary": "plain string thought",
},
{"type": "reasoning", "encrypted_content": wrapped, "summary": "plain string thought"},
{
"type": "reasoning",
"encrypted_content": wrapped,
"content": [{"type": "output_text", "text": "in content"}],
},
{
"type": "reasoning",
"encrypted_content": wrapped,
"summary": "",
"content": [],
},
{"type": "reasoning", "encrypted_content": wrapped, "summary": "", "content": []},
]
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input)
assert request_input == [
{"type": "reasoning", "summary": "plain string thought"},
{
"type": "reasoning",
"content": [{"type": "output_text", "text": "in content"}],
},
{"type": "reasoning", "content": [{"type": "output_text", "text": "in content"}]},
]
def test_leaves_input_untouched_when_no_encrypted_reasoning(self):
request_input = [
{"role": "user", "content": "first turn"},
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "no blob here"}],
},
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "no blob here"}]},
{"role": "user", "content": "second turn"},
]
before = [dict(item) for item in request_input]
@ -1877,9 +1724,7 @@ class TestStripEncryptedReasoningFromInput:
def _cross_group_request_kwargs():
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA-blob", "deployment-a"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a")
return {
"litellm_metadata": {},
"input": [
@ -1935,9 +1780,7 @@ async def test_affinity_strips_encrypted_reasoning_when_routed_to_another_model_
assert result is routed_pool
assert "_encrypted_content_affinity_pinned" not in request_kwargs
assert (
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
)
assert request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
assert request_kwargs["input"] is original_input
assert [item.get("type") or item["role"] for item in original_input] == [
"user",
@ -1949,10 +1792,7 @@ async def test_affinity_strips_encrypted_reasoning_when_routed_to_another_model_
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "scattering"}],
}
assert not any(
isinstance(item, dict) and item.get("encrypted_content")
for item in original_input
)
assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in original_input)
@pytest.mark.asyncio
@ -1973,9 +1813,7 @@ async def test_affinity_fails_fast_within_the_origins_own_group():
"https://account-a.openai.azure.com/", "key-a", model_name="gpt-reasoning-tier"
)
mock_router = _make_router_mock_with_cooldown(
originating,
cooldown_entries=[],
routed_group_model_ids=["deployment-a", "deployment-a2"],
originating, cooldown_entries=[], routed_group_model_ids=["deployment-a", "deployment-a2"]
)
check = EncryptedContentAffinityCheck(router=mock_router)
sibling_pool = [
@ -2018,18 +1856,12 @@ async def test_affinity_does_not_strip_when_group_is_spelled_differently_but_sam
EncryptedContentAffinityCheck,
)
originating = _make_originating_mock(
None, "key-a", model_name="openai/gpt-5.4-mini"
)
originating = _make_originating_mock(None, "key-a", model_name="openai/gpt-5.4-mini")
mock_router = _make_router_mock_with_cooldown(
originating,
cooldown_entries=[],
routed_group_model_ids=["deployment-mini-a", "deployment-mini-b"],
originating, cooldown_entries=[], routed_group_model_ids=["deployment-mini-a", "deployment-mini-b"]
)
check = EncryptedContentAffinityCheck(router=mock_router)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA-blob", "deployment-mini-a"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-mini-a")
sibling_pool = [
{
"model_info": {"id": "deployment-mini-b"},
@ -2080,18 +1912,12 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes(
EncryptedContentAffinityCheck,
)
originating = _make_originating_mock(
None, "key-a", model_name="model_name_teamA_uuid"
)
originating = _make_originating_mock(None, "key-a", model_name="model_name_teamA_uuid")
mock_router = _make_router_mock_with_cooldown(
originating,
cooldown_entries=[],
routed_group_model_ids=["deployment-team-a", "deployment-team-b"],
originating, cooldown_entries=[], routed_group_model_ids=["deployment-team-a", "deployment-team-b"]
)
check = EncryptedContentAffinityCheck(router=mock_router)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
"gAAAAA-blob", "deployment-team-a"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-team-a")
sibling_pool = [
{
"model_info": {"id": "deployment-team-b"},
@ -2125,6 +1951,4 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes(
)
assert request_kwargs["input"][1].get("encrypted_content")
mock_router.get_candidate_model_ids_for_route.assert_called_once_with(
model="team-public-model", team_id="teamA"
)
mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA")

View file

@ -2967,9 +2967,15 @@ def test_gemini_embedding_2_ga_in_cost_map():
assert info.get("mode") == "embedding"
assert info.get("supports_multimodal") is True
assert info.get("input_cost_per_token") == 2e-07
assert info.get("input_cost_per_image") == 0.00012
assert info.get("input_cost_per_audio_per_second") == 0.00016
assert info.get("input_cost_per_video_per_second") == 0.00079
if key.endswith("gemini-embedding-2"):
assert info.get("input_cost_per_audio_token") == 6.5e-06
assert info.get("input_cost_per_image_token") == 4.5e-07
assert info.get("input_cost_per_token_batches") == 1e-07
assert info.get("input_cost_per_video_token") == 1.2e-05
else:
assert info.get("input_cost_per_image") == 0.00012
assert info.get("input_cost_per_audio_per_second") == 0.00016
assert info.get("input_cost_per_video_per_second") == 0.00079
if provider in ("vertex_ai-embedding-models", "vertex_ai"):
assert (
info.get("uses_embed_content") is True
@ -6503,3 +6509,4 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
assert snapshot["litellm_call_id"]
assert snapshot["response_cost"] is not None
assert snapshot["api_base"]