From d992d246790b6f36ecd903cbf0363c7488e4e9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=80=A1=E9=9C=96=20=C2=B7=20Yilin=20Li?= <2747028274@qq.com> Date: Fri, 18 Sep 2026 11:56:24 +0800 Subject: [PATCH] test: align Gemini embedding pricing assertions --- .../litellm_core_utils/llm_cost_calc/utils.py | 12 +- .../batch_embed_content_transformation.py | 41 ++- ...odel_prices_and_context_window_backup.json | 23 +- .../encrypted_content_affinity_check.py | 62 +--- model_prices_and_context_window.json | 23 +- model_prices_and_context_window.schema.json | 4 - .../auth/test_auth_object_prefetch.py | 12 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 32 +- ...test_batch_embed_content_transformation.py | 123 ++----- .../test_encrypted_content_affinity_check.py | 326 ++++-------------- tests/test_litellm/test_utils.py | 13 +- 11 files changed, 177 insertions(+), 494 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 09f8a726ba2..e5977ca4156 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -921,16 +921,12 @@ def _calculate_input_cost( prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost ### AUDIO COST - if prompt_tokens_details["audio_tokens"] and not ( - prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None - ): + if prompt_tokens_details["audio_tokens"]: audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) ### IMAGE TOKEN COST - if prompt_tokens_details["image_tokens"] and not ( - prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None - ): + if prompt_tokens_details["image_tokens"]: # For image token costs: # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. image_token_cost_key = "input_cost_per_image_token" @@ -939,9 +935,7 @@ def _calculate_input_cost( prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]) ### VIDEO TOKEN COST - if prompt_tokens_details["video_tokens"] and not ( - prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None - ): + if prompt_tokens_details["video_tokens"]: video_token_cost_key = "input_cost_per_video_token" if model_info.get(video_token_cost_key) is None: video_token_cost_key = "input_cost_per_token" diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index d669acecfd9..e7fd9a0d08b 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -298,6 +298,8 @@ def transform_openai_input_gemini_embed_content( _IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"}) +_VIDEO_TOKENS_PER_SECOND: Final = 258.0 +_AUDIO_TOKENS_PER_SECOND: Final = 32.0 _usage_metadata_adapter: Final = TypeAdapter(UsageMetadata) @@ -337,12 +339,11 @@ def _is_image_element( return False -def _is_image_only_input( +def _count_input_images( input: GeminiEmbeddingInput, resolved_files: Mapping[str, Mapping[str, str]], -) -> bool: - elements: Final = _flatten_input(input) - return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements) +) -> int: + return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files)) def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: @@ -371,29 +372,30 @@ def _usage_from_embed_content_response( total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or () - if not details: - return Usage( - prompt_tokens=prompt_tokens, - total_tokens=total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=0, - image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0, - ), - ) - text_tokens: Final = _tokens_for_modality(details, "TEXT") audio_tokens: Final = _tokens_for_modality(details, "AUDIO") - image_tokens: Final = _tokens_for_modality(details, "IMAGE") video_tokens: Final = _tokens_for_modality(details, "VIDEO") + image_count: Final = _count_input_images(input, resolved_files) + + video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0 + audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0 + + # generic_cost_per_token rewrites text_tokens to the full prompt minus + # other modalities when both text_tokens and image_count are zero. For + # video, that misallocates video tokens to text; a 1-token floor sidesteps + # the rewrite and keeps billing on input_cost_per_video_per_second. + needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0 + resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens return Usage( prompt_tokens=prompt_tokens, total_tokens=total_tokens, prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=text_tokens, + text_tokens=resolved_text_tokens, audio_tokens=audio_tokens, - image_tokens=image_tokens, - video_tokens=video_tokens, + image_count=image_count, + video_length_seconds=video_length_seconds, + audio_length_seconds=audio_length_seconds, ), ) @@ -413,7 +415,8 @@ def process_embed_content_response( model_response: EmbeddingResponse to populate model: Model name response_json: Raw JSON response from embedContent endpoint - resolved_files: Mapping of file references to resolved metadata + resolved_files: Mapping of file references (files/abc) to {mime_type, uri}, + used to bill resolved image references at the per-image rate Returns: EmbeddingResponse with single embedding diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d1b3a87531b..2220d0e1fe5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25380,11 +25380,10 @@ "uses_embed_content": true }, "gemini-embedding-2": { - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image_token": 4.5e-07, + "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_image": 0.00012, "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_per_second": 0.00079, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25411,11 +25410,10 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2": { - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image_token": 4.5e-07, + "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_image": 0.00012, "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_per_second": 0.00079, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25470,11 +25468,10 @@ "tpm": 10000000 }, "gemini/gemini-embedding-2": { - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image_token": 4.5e-07, + "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_image": 0.00012, "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_per_second": 0.00079, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, @@ -65274,4 +65271,4 @@ "supports_response_schema": true, "supports_vision": false } -} \ No newline at end of file +} diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 0fc8ac8c91d..cdd70e6baf2 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -30,8 +30,8 @@ This pre-call check is responsible only for the routing decision: it reads the e the matching deployment. Safe to enable globally: -- Only pins requests carrying encoded markers in ``input`` or assistant ``reasoning_items``. -- No effect on embedding models or first-time deployment selection. +- Only activates when encoded markers appear in the request ``input``. +- No effect on embedding models, chat completions, or first-time requests. - No quota reduction -- first requests are fully load balanced. - No cache required. """ @@ -71,8 +71,8 @@ class _SupportsActiveCooldowns(Protocol): class EncryptedContentAffinityCheck(CustomLogger): """ - Routes follow-up Responses API and bridged chat requests to the deployment - that produced the encrypted output items they reference. + Routes follow-up Responses API requests to the deployment that produced + the encrypted output items they reference. The ``model_id`` is decoded directly from the litellm-encoded item IDs – no caching or TTL management needed. @@ -184,27 +184,6 @@ class EncryptedContentAffinityCheck(CustomLogger): None, ) - @staticmethod - def _extract_model_id_from_messages( - messages: list[AllMessageValues] | None, - ) -> str | None: - if messages is None: - return None - return next( - ( - model_id - for message in messages - if message.get("role") == "assistant" - if ( - model_id := EncryptedContentAffinityCheck._extract_model_id_from_input( - message.get("reasoning_items") - ) - ) - is not None - ), - None, - ) - @staticmethod def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: @@ -218,10 +197,7 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _request_team_id(request_kwargs: Mapping[str, object]) -> str | None: - containers: Final = ( - request_kwargs.get("metadata"), - request_kwargs.get("litellm_metadata"), - ) + containers: Final = (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) team_ids: Final = (c.get("user_api_key_team_id") for c in containers if isinstance(c, Mapping)) return next((tid for tid in team_ids if isinstance(tid, str)), None) @@ -316,7 +292,7 @@ class EncryptedContentAffinityCheck(CustomLogger): remaining cooldown window) so OpenAI-compatible clients back off and retry after the deployment is eligible again. """ - routing_kwargs: Final = request_kwargs if request_kwargs is not None else {} + request_kwargs = request_kwargs or {} typed_healthy_deployments: Final = cast(list[dict], healthy_deployments) if not self._is_enabled_for_model_group(model): return typed_healthy_deployments @@ -328,20 +304,14 @@ class EncryptedContentAffinityCheck(CustomLogger): # completions / embeddings, which breaks tag-based routing because # _get_metadata_variable_name_from_kwargs would pick "litellm_metadata" # over "metadata" where tags are actually stored. - if "litellm_metadata" in routing_kwargs or messages is not None: - metadata_key: Final = "litellm_metadata" if "litellm_metadata" in routing_kwargs else "metadata" - routing_kwargs[metadata_key] = { - **(routing_kwargs.get(metadata_key) or {}), - "encrypted_content_affinity_enabled": True, - } + if "litellm_metadata" in request_kwargs: + request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True - request_input: Final = routing_kwargs.get("input") - anthropic_messages: Final = messages or routing_kwargs.get("messages") - model_id: Final = ( - self._extract_model_id_from_input(request_input) - or self._extract_model_id_from_messages(messages) - or self._extract_model_id_from_anthropic_messages(anthropic_messages) - ) + request_input: Final = request_kwargs.get("input") + anthropic_messages: Final = messages or request_kwargs.get("messages") + model_id: Final = self._extract_model_id_from_input( + request_input + ) or self._extract_model_id_from_anthropic_messages(anthropic_messages) if not model_id: return typed_healthy_deployments @@ -359,7 +329,7 @@ class EncryptedContentAffinityCheck(CustomLogger): "EncryptedContentAffinityCheck: pinning -> deployment=%s", model_id, ) - routing_kwargs["_encrypted_content_affinity_pinned"] = True + request_kwargs["_encrypted_content_affinity_pinned"] = True return [deployment] # Follow-up switched model_name (LIT-2531): pin by Azure resource instead. @@ -374,7 +344,7 @@ class EncryptedContentAffinityCheck(CustomLogger): model_id, len(boundary_matches), ) - routing_kwargs["_encrypted_content_affinity_pinned"] = True + request_kwargs["_encrypted_content_affinity_pinned"] = True return boundary_matches # The origin cannot serve this turn's routed group and no peer shares the boundary, so its @@ -388,7 +358,7 @@ class EncryptedContentAffinityCheck(CustomLogger): # dispatch rather than returning distinguishable responses. Only a genuine same-group member # that is currently unavailable falls through to the fail-fast, preserving the cooldown contract. routed_group_model_ids: Final = ( - self._routed_group_candidate_model_ids(routing_kwargs, model) if originating is not None else frozenset() + self._routed_group_candidate_model_ids(request_kwargs, model) if originating is not None else frozenset() ) if str(model_id) not in routed_group_model_ids: verbose_router_logger.debug( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d1b3a87531b..2220d0e1fe5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25380,11 +25380,10 @@ "uses_embed_content": true }, "gemini-embedding-2": { - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image_token": 4.5e-07, + "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_image": 0.00012, "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_per_second": 0.00079, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25411,11 +25410,10 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2": { - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image_token": 4.5e-07, + "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_image": 0.00012, "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_per_second": 0.00079, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25470,11 +25468,10 @@ "tpm": 10000000 }, "gemini/gemini-embedding-2": { - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image_token": 4.5e-07, + "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_image": 0.00012, "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_per_second": 0.00079, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, @@ -65274,4 +65271,4 @@ "supports_response_schema": true, "supports_vision": false } -} \ No newline at end of file +} diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index eff3f192b3c..d1ac3e67b2b 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -375,10 +375,6 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, - "input_cost_per_video_token": { - "type": "number", - "minimum": 0 - }, "input_dbu_cost_per_token": { "type": "number", "minimum": 0 diff --git a/tests/proxy_behavior/auth/test_auth_object_prefetch.py b/tests/proxy_behavior/auth/test_auth_object_prefetch.py index cfa958500af..e2d947f4284 100644 --- a/tests/proxy_behavior/auth/test_auth_object_prefetch.py +++ b/tests/proxy_behavior/auth/test_auth_object_prefetch.py @@ -22,11 +22,6 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache pytestmark = pytest.mark.asyncio(loop_scope="session") -def _frozen_cache() -> UserApiKeyCache: - """The org entries carry a 5s TTL; a frozen clock keeps a slow first call from expiring them mid-test.""" - return UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1_000_000.0), redis_cache=None) - - def _dead_db() -> MagicMock: prisma = MagicMock(name="prisma_client") prisma.db.query_first = AsyncMock(return_value=None) @@ -63,10 +58,9 @@ async def test_join_binds_the_membership_to_the_requested_team(prisma): data={"user_id": user_id, "team_id": team_b, "litellm_budget_table": {"connect": {"budget_id": f"b-{run}"}}} ) - cache = _frozen_cache() + cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None) refs = AuthObjectRefs(user_id=user_id, team_id=team_a, membership_user_id=user_id, organization_id=org_id) await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma) - assert cache.in_memory_cache.get_cache(f"org_id:{org_id}") is not None dead_db = _dead_db() membership = await get_team_membership( @@ -106,7 +100,7 @@ async def test_join_reads_team_model_aliases_from_the_mapped_column(prisma): where={"team_id": team_id}, include={"litellm_model_table": True} ) - cache = _frozen_cache() + cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None) refs = AuthObjectRefs(user_id=None, team_id=team_id, membership_user_id=None, organization_id=None) await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma) @@ -150,7 +144,7 @@ async def test_join_reads_null_nested_lists_the_way_prisma_does(prisma): where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}}, include={"litellm_budget_table": True} ) - cache = _frozen_cache() + cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None) refs = AuthObjectRefs(user_id=user_id, team_id=team_id, membership_user_id=user_id, organization_id=None) await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index a6c3b88cc36..cbe6fe198c9 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -74,36 +74,6 @@ def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, s assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4]) -def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: - model_info: ModelInfo = { - "key": "gemini-embedding-2", - "max_tokens": None, - "max_input_tokens": None, - "max_output_tokens": None, - "input_cost_per_token": 2e-7, - "input_cost_per_audio_token": 6.5e-6, - "input_cost_per_audio_per_second": 0.00016, - "output_cost_per_token": 0.0, - "litellm_provider": "vertex_ai", - "mode": "embedding", - } - usage = Usage( - prompt_tokens=64, - completion_tokens=0, - total_tokens=64, - prompt_tokens_details=PromptTokensDetailsWrapper( - audio_tokens=64, - audio_length_seconds=2, - ), - ) - prompt_cost, _ = generic_cost_per_token( - model="gemini-embedding-2", - usage=usage, - custom_llm_provider="vertex_ai", - model_info=model_info, - ) - assert prompt_cost == pytest.approx(2 * 0.00016) - def test_missing_cache_read_uses_off_peak_input_rate(): from datetime import datetime, timezone @@ -5176,4 +5146,4 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output( assert breakdown.reasoning_cost == pytest.approx(20 * info["output_cost_per_token"]) assert completion_cost == pytest.approx( 30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"] - ) \ No newline at end of file + ) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 5dfeac6f469..86b3f0976ab 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -10,7 +10,6 @@ Covers: import pytest -import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, @@ -23,19 +22,11 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject from litellm.types.utils import EmbeddingResponse + IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII" GCS_URL = "gs://my-bucket/image.png" -@pytest.fixture(autouse=True) -def _local_model_cost_map(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - litellm.get_model_info.cache_clear() - yield - litellm.get_model_info.cache_clear() - - class TestIsMultimodalInput: def test_text_only_string(self): assert _is_multimodal_input("hello world") is False @@ -333,7 +324,7 @@ class TestProcessEmbedContentResponseUsage: ) assert result.usage.prompt_tokens == 258 assert result.usage.total_tokens == 258 - assert result.usage.prompt_tokens_details.image_tokens == 258 + assert result.usage.prompt_tokens_details.image_count == 1 prompt_cost, _ = generic_cost_per_token( model=self.MODEL, @@ -367,7 +358,7 @@ class TestProcessEmbedContentResponseUsage: ) assert prompt_cost > 0 - def test_video_modality_preserves_token_count(self): + def test_video_modality_derives_seconds_and_text_floor(self): response_json = { "embedding": {"values": [0.1]}, "usageMetadata": { @@ -383,8 +374,10 @@ class TestProcessEmbedContentResponseUsage: response_json=response_json, ) assert result.usage.prompt_tokens == 516 - assert result.usage.prompt_tokens_details.video_tokens == 516 - assert result.usage.prompt_tokens_details.text_tokens == 0 + assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx( + 2.0 + ) + assert result.usage.prompt_tokens_details.text_tokens == 1 def test_missing_usage_metadata_does_not_estimate_from_base64(self): response_json = {"embedding": {"values": [0.1, 0.2]}} @@ -407,7 +400,8 @@ class TestProcessEmbedContentResponseUsage: ) assert result.usage.prompt_tokens > 0 - def test_file_reference_image_billed_per_image_token_rate(self): + def test_file_reference_image_billed_per_image_not_text(self): + """files/... image refs must bill per-image, not at the text token rate.""" response_json = { "embedding": {"values": [0.1, 0.2, 0.3]}, "usageMetadata": { @@ -428,7 +422,7 @@ class TestProcessEmbedContentResponseUsage: } }, ) - assert result.usage.prompt_tokens_details.image_tokens == 258 + assert result.usage.prompt_tokens_details.image_count == 1 assert result.usage.prompt_tokens_details.text_tokens == 0 prompt_cost, _ = generic_cost_per_token( @@ -436,10 +430,10 @@ class TestProcessEmbedContentResponseUsage: usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(258 * 4.5e-7) + assert prompt_cost == pytest.approx(0.00012) def test_file_reference_non_image_not_counted_as_image(self): - """A files/... ref resolving to a non-image mime keeps audio token billing.""" + """A files/... ref resolving to a non-image mime must not be image-counted.""" response_json = { "embedding": {"values": [0.1, 0.2]}, "usageMetadata": { @@ -460,18 +454,21 @@ class TestProcessEmbedContentResponseUsage: } }, ) + assert result.usage.prompt_tokens_details.image_count == 0 assert result.usage.prompt_tokens_details.audio_tokens == 64 - assert result.usage.prompt_tokens_details.image_tokens == 0 + assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx( + 2.0 + ) prompt_cost, _ = generic_cost_per_token( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(64 * 6.5e-6) + assert prompt_cost == pytest.approx(2.0 * 0.00016) def test_video_plus_audio_does_not_double_bill_text(self): - """Video and audio responses are billed from their respective token counts.""" + """Video+audio responses must not get video tokens reassigned to text.""" response_json = { "embedding": {"values": [0.1]}, "usageMetadata": { @@ -489,84 +486,18 @@ class TestProcessEmbedContentResponseUsage: model=self.MODEL, response_json=response_json, ) - assert result.usage.prompt_tokens_details.text_tokens == 0 - assert result.usage.prompt_tokens_details.video_tokens == 516 - assert result.usage.prompt_tokens_details.audio_tokens == 64 + assert result.usage.prompt_tokens_details.text_tokens == 1 + assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx( + 2.0 + ) + assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx( + 2.0 + ) prompt_cost, _ = generic_cost_per_token( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6) - - def test_image_without_modality_details_uses_image_rate(self): - response_json = { - "embedding": {"values": [0.1]}, - "usageMetadata": { - "promptTokenCount": 258, - "totalTokenCount": 258, - }, - } - result = process_embed_content_response( - input=IMAGE_DATA_URI, - model_response=EmbeddingResponse(), - model=self.MODEL, - response_json=response_json, - ) - assert result.usage.prompt_tokens_details.image_tokens == 258 - assert result.usage.prompt_tokens_details.text_tokens == 0 - - prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, - usage=result.usage, - custom_llm_provider="vertex_ai", - ) - assert prompt_cost == pytest.approx(258 * 4.5e-7) - - def test_mixed_text_and_image_without_modality_details_not_billed_as_image(self): - response_json = { - "embedding": {"values": [0.1]}, - "usageMetadata": { - "promptTokenCount": 270, - "totalTokenCount": 270, - }, - } - result = process_embed_content_response( - input=["a short caption", IMAGE_DATA_URI], - model_response=EmbeddingResponse(), - model=self.MODEL, - response_json=response_json, - ) - assert result.usage.prompt_tokens_details.image_tokens == 0 - - prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, - usage=result.usage, - custom_llm_provider="vertex_ai", - ) - assert prompt_cost == pytest.approx(270 * 2e-7) - - def test_text_without_modality_details_uses_text_rate(self): - response_json = { - "embedding": {"values": [0.1]}, - "usageMetadata": { - "promptTokenCount": 12, - "totalTokenCount": 12, - }, - } - result = process_embed_content_response( - input="a short caption", - model_response=EmbeddingResponse(), - model=self.MODEL, - response_json=response_json, - ) - assert result.usage.prompt_tokens_details.text_tokens == 0 - assert result.usage.prompt_tokens_details.image_tokens == 0 - - prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, - usage=result.usage, - custom_llm_provider="vertex_ai", - ) - assert prompt_cost == pytest.approx(12 * 2e-7) + # 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4 + assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016) diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 77158b5aeeb..ea8e2eacaa6 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -18,8 +18,9 @@ The mechanism works without any cache and supports two encoding strategies: import time from unittest.mock import AsyncMock, patch -import litellm import pytest + +import litellm from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIResponse @@ -65,9 +66,7 @@ class TestEncryptedItemIdCodec: def test_roundtrip(self): model_id = "deployment-1" original_item_id = "rs_abc123def456" - encoded = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_item_id - ) + encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id) assert encoded.startswith("encitem_") decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded) assert decoded is not None @@ -78,9 +77,7 @@ class TestEncryptedItemIdCodec: """Decoding must succeed even if base64 padding (=) was stripped in transit.""" model_id = "gpt-5.1-codex-openai-2" original_item_id = "rs_0efb96cb222403210069a01d5d52588196a9dc394ffdb89d00" - encoded = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_item_id - ) + encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id) # Strip any trailing '=' to simulate what happens in transit stripped = encoded.rstrip("=") decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(stripped) @@ -97,9 +94,7 @@ class TestEncryptedItemIdCodec: """item_id values containing ';' must survive the roundtrip.""" model_id = "deployment-1" original_item_id = "rs_part1;part2;part3" - encoded = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_item_id - ) + encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id) decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded) assert decoded is not None assert decoded["item_id"] == original_item_id @@ -115,11 +110,7 @@ class TestUpdateEncryptedContentItemIds: {"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}, ], } - result = ( - ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( - response, model_id - ) - ) + result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, model_id) # Plain message item untouched assert result["output"][0]["id"] == "msg_abc" # Reasoning item with encrypted_content gets encoded @@ -130,16 +121,8 @@ class TestUpdateEncryptedContentItemIds: assert decoded["item_id"] == "rs_xyz" def test_no_op_when_model_id_is_none(self): - response = { - "output": [ - {"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"} - ] - } - result = ( - ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( - response, None - ) - ) + response = {"output": [{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}]} + result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, None) assert result["output"][0]["id"] == "rs_xyz" @@ -148,9 +131,7 @@ class TestEncryptedContentWrapping: """Test wrapping encrypted_content with model_id metadata.""" model_id = "deployment-1" original_content = "gAAAAABpnW_yEYmSNEyOG_original_encrypted_data" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) assert wrapped.startswith("litellm_enc:") assert wrapped != original_content @@ -167,9 +148,7 @@ class TestEncryptedContentWrapping: ( model_id, content, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id( - plain_content - ) + ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(plain_content) assert model_id is None assert content == plain_content @@ -186,11 +165,7 @@ class TestEncryptedContentWrapping: }, ], } - result = ( - ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( - response, model_id - ) - ) + result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, model_id) assert result["output"][0].get("encrypted_content") is None wrapped = result["output"][1]["encrypted_content"] assert wrapped.startswith("litellm_enc:") @@ -207,19 +182,13 @@ class TestRestoreEncryptedContentItemIds: def test_restores_encoded_ids(self): model_id = "deployment-1" original_id = "rs_encrypted_item_456" - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_id - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_id) request_input = [ {"type": "message", "id": "msg_abc123", "role": "assistant"}, {"type": "reasoning", "id": encoded_id, "encrypted_content": "secret"}, ] - restored = ( - ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - request_input - ) - ) + restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input) assert restored[0]["id"] == "msg_abc123" assert restored[1]["id"] == original_id @@ -227,33 +196,21 @@ class TestRestoreEncryptedContentItemIds: """Test that wrapped encrypted_content is unwrapped before forwarding.""" model_id = "deployment-1" original_content = "gAAAAABpnW_yEYmSNEyOG_original" - wrapped_content = ( - ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) - ) + wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) request_input = [ {"type": "reasoning", "encrypted_content": wrapped_content}, ] - restored = ( - ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - request_input - ) - ) + restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input) assert restored[0]["encrypted_content"] == original_content def test_no_op_for_plain_string_input(self): - result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - "Hello world" - ) + result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input("Hello world") assert result == "Hello world" def test_no_op_for_unencoded_ids(self): request_input = [{"type": "message", "id": "msg_plain"}] - result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - request_input - ) + result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input) assert result[0]["id"] == "msg_plain" @@ -280,9 +237,7 @@ async def test_encrypted_content_affinity_tracks_and_routes(): "id": "msg_abc123", "status": "completed", "role": "assistant", - "content": [ - {"type": "output_text", "text": "Hello!", "annotations": []} - ], + "content": [{"type": "output_text", "text": "Hello!", "annotations": []}], }, { "type": "reasoning", @@ -475,9 +430,7 @@ async def test_encrypted_content_affinity_bypasses_rpm_limits(): # Extract encoded item ID from the first response output encoded_item_id = _extract_encoded_item_id(first_response) - assert encoded_item_id.startswith("encitem_"), ( - f"Expected encitem_... but got {encoded_item_id!r}" - ) + assert encoded_item_id.startswith("encitem_"), f"Expected encitem_... but got {encoded_item_id!r}" # Follow-up with the encoded item ID — should pin to same deployment second_response = await router.aresponses( @@ -625,17 +578,13 @@ async def test_encrypted_content_affinity_with_wrapped_content_no_id(): if hasattr(first_item, "encrypted_content") else first_item.get("encrypted_content") ) - assert wrapped_content.startswith("litellm_enc:"), ( - f"Expected wrapped content but got {wrapped_content[:50]}..." - ) + assert wrapped_content.startswith("litellm_enc:"), f"Expected wrapped content but got {wrapped_content[:50]}..." # Verify we can extract model_id from wrapped content ( extracted_model_id, _, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id( - wrapped_content - ) + ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped_content) assert extracted_model_id == first_model_id # Second request: use wrapped encrypted_content WITHOUT an ID (Codex behavior) @@ -661,13 +610,9 @@ def test_encrypted_content_wrapping_preserves_original_content(): This is critical for streaming responses where content must round-trip correctly. """ model_id = "test-deployment-1" - original_encrypted_content = ( - "gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/" - ) + original_encrypted_content = "gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_encrypted_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_encrypted_content, model_id) assert wrapped.startswith("litellm_enc:") assert wrapped != original_encrypted_content @@ -688,9 +633,7 @@ def test_encrypted_content_wrapping_with_multiple_semicolons(): model_id = "deployment-with-semicolons" original_content = "gAAAAAB;some;content;with;semicolons" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) ( extracted_model_id, @@ -761,9 +704,7 @@ async def test_encrypted_content_affinity_preserves_litellm_metadata_for_respons request_kwargs=request_kwargs, ) - assert ( - request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True - ) + assert request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True assert request_kwargs["litellm_metadata"]["model_info"] == {"id": "dep-1"} @@ -774,9 +715,7 @@ def test_encrypted_content_wrapping_empty_string(): model_id = "test-deployment" original_content = "" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) assert wrapped.startswith("litellm_enc:") @@ -1129,9 +1068,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance(): "api_key": "fake-azure-resource-key-a", } - pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key( - pydantic_params - ) + pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key(pydantic_params) plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params) assert pydantic_key is not None @@ -1158,18 +1095,8 @@ def test_boundary_key_rejects_non_dict_like_inputs(): for bad in (None, [], "not a dict", 42, object()): assert EncryptedContentAffinityCheck._encryption_boundary_key(bad) is None - assert ( - EncryptedContentAffinityCheck._encryption_boundary_key( - {"api_base": "", "api_key": "k"} - ) - is None - ) - assert ( - EncryptedContentAffinityCheck._encryption_boundary_key( - {"api_base": "https://x"} - ) - is None - ) + assert EncryptedContentAffinityCheck._encryption_boundary_key({"api_base": "", "api_key": "k"}) is None + assert EncryptedContentAffinityCheck._encryption_boundary_key({"api_base": "https://x"}) is None # --------------------------------------------------------------------------- @@ -1205,12 +1132,8 @@ def _make_router_mock_with_cooldown( mock_router = MagicMock() mock_router.get_deployment.return_value = originating - mock_router.cooldown_cache.async_get_active_cooldowns = AsyncMock( - return_value=list(cooldown_entries or []) - ) - mock_router.get_candidate_model_ids_for_route.return_value = frozenset( - routed_group_model_ids or [] - ) + mock_router.cooldown_cache.async_get_active_cooldowns = AsyncMock(return_value=list(cooldown_entries or [])) + mock_router.get_candidate_model_ids_for_route.return_value = frozenset(routed_group_model_ids or []) return mock_router @@ -1245,9 +1168,7 @@ async def test_affinity_raises_service_unavailable_when_origin_cooled_for_non_42 ) check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a-cooled", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-cooled", "rs_test") healthy_only_b = [ { "model_info": {"id": "deployment-b"}, @@ -1309,9 +1230,7 @@ async def test_affinity_raises_rate_limit_with_retry_after_when_origin_cooled_fo ) check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a-cooled-429", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-cooled-429", "rs_test") healthy_only_b = [ { "model_info": {"id": "deployment-b"}, @@ -1356,15 +1275,11 @@ async def test_affinity_raises_service_unavailable_when_origin_filtered_without_ originating = _make_originating_mock("https://account-a.openai.azure.com/", "key-a") mock_router = _make_router_mock_with_cooldown( - originating, - cooldown_entries=[], - routed_group_model_ids=["deployment-a-filtered", "deployment-b"], + originating, cooldown_entries=[], routed_group_model_ids=["deployment-a-filtered", "deployment-b"] ) check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a-filtered", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-filtered", "rs_test") healthy_only_b = [ { "model_info": {"id": "deployment-b"}, @@ -1412,9 +1327,7 @@ async def test_affinity_strips_and_dispatches_when_origin_is_unknown_or_removed( mock_router.get_deployment.return_value = None check = EncryptedContentAffinityCheck(router=mock_router) - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA-blob", "deployment-removed" - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-removed") routed_pool = [ { "model_info": {"id": "deployment-b"}, @@ -1447,10 +1360,7 @@ async def test_affinity_strips_and_dispatches_when_origin_is_unknown_or_removed( ) assert result is routed_pool - assert not any( - isinstance(item, dict) and item.get("encrypted_content") - for item in request_kwargs["input"] - ) + assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in request_kwargs["input"]) mock_router.get_candidate_model_ids_for_route.assert_not_called() @@ -1476,9 +1386,7 @@ async def test_affinity_does_not_raise_when_boundary_peer_available(): mock_router.get_deployment.return_value = originating check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a", "rs_test") peer = { "model_info": {"id": "deployment-a-peer"}, "litellm_params": { @@ -1522,9 +1430,7 @@ async def test_model_group_affinity_config_enables_encrypted_content_affinity(): }, target_deployment, ] - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-b", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test") request_kwargs = { "input": [{"type": "reasoning", "id": encoded_id}], "litellm_metadata": {}, @@ -1568,9 +1474,7 @@ async def test_model_group_affinity_config_does_not_disable_global_encrypted_con }, target_deployment, ] - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-b", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test") request_kwargs = { "input": [{"type": "reasoning", "id": encoded_id}], "litellm_metadata": {}, @@ -1632,15 +1536,9 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen try: callbacks = router.optional_callbacks or [] - deployment_callback = next( - cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck) - ) - encrypted_content_callback = next( - cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck) - ) - assert callbacks.index(encrypted_content_callback) < callbacks.index( - deployment_callback - ) + deployment_callback = next(cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck)) + encrypted_content_callback = next(cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck)) + assert callbacks.index(encrypted_content_callback) < callbacks.index(deployment_callback) assert encrypted_content_callback.enable_global_affinity is False cache_key = DeploymentAffinityCheck.get_affinity_cache_key( @@ -1652,9 +1550,7 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen value={"model_id": "deployment-a"}, ttl=60, ) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-b", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test") request_kwargs = { "input": [ { @@ -1675,13 +1571,11 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen ) assert after_deployment_affinity == [deployment_a, deployment_b] - after_encrypted_content_affinity = ( - await encrypted_content_callback.async_filter_deployments( - model=model_group, - healthy_deployments=after_deployment_affinity, - messages=None, - request_kwargs=request_kwargs, - ) + after_encrypted_content_affinity = await encrypted_content_callback.async_filter_deployments( + model=model_group, + healthy_deployments=after_deployment_affinity, + messages=None, + request_kwargs=request_kwargs, ) assert after_encrypted_content_affinity == [deployment_b] @@ -1700,14 +1594,8 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu """ check = EncryptedContentAffinityCheck() deployments = [ - { - "model_info": {"id": "openai-org-a"}, - "litellm_params": {"model": "openai/gpt-5.1"}, - }, - { - "model_info": {"id": "openai-org-b"}, - "litellm_params": {"model": "openai/gpt-5.1"}, - }, + {"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + {"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}}, ] request_kwargs = {"model": "gpt-5.1"} @@ -1723,23 +1611,14 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu def _bridge_replayed_anthropic_messages(minted_by: str) -> list: - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA_turn_one", minted_by - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by) return [ {"role": "user", "content": "Solve the zebra puzzle"}, { "role": "assistant", "content": [ - { - "type": "thinking", - "thinking": "Anthropic minted this one", - "signature": "ErcCCpIBCBEYAipA", - }, - { - "type": "redacted_thinking", - "data": f"litellm_encrypted_reasoning:{wrapped}", - }, + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, { "type": "thinking", "thinking": "The bridge packed this one", @@ -1766,12 +1645,7 @@ async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_ originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"] ) check = EncryptedContentAffinityCheck(router=mock_router) - routed_pool = [ - { - "model_info": {"id": "openai-org-b"}, - "litellm_params": {"model": "openai/gpt-5-nano"}, - } - ] + routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}] messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a") assistant_content = messages[1]["content"] request_kwargs = {"model": "gpt-5.1"} @@ -1787,26 +1661,16 @@ async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_ assert "_encrypted_content_affinity_pinned" not in request_kwargs assert messages[1]["content"] is assistant_content assert assistant_content == [ - { - "type": "thinking", - "thinking": "Anthropic minted this one", - "signature": "ErcCCpIBCBEYAipA", - }, + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, {"type": "text", "text": "The zebra owner lives in the green house."}, ] - assert all( - block["signature"] for block in assistant_content if block["type"] == "thinking" - ) + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") class TestStripEncryptedReasoningFromInput: def test_keeps_summary_and_drops_encrypted_content_and_id(self): - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA-blob", "deployment-a" - ) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a", "rs_1" - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a", "rs_1") request_input = [ {"role": "user", "content": "first turn"}, { @@ -1832,43 +1696,26 @@ class TestStripEncryptedReasoningFromInput: ] def test_keeps_string_form_summary_when_stripping(self): - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA-blob", "deployment-a" - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") request_input = [ - { - "type": "reasoning", - "encrypted_content": wrapped, - "summary": "plain string thought", - }, + {"type": "reasoning", "encrypted_content": wrapped, "summary": "plain string thought"}, { "type": "reasoning", "encrypted_content": wrapped, "content": [{"type": "output_text", "text": "in content"}], }, - { - "type": "reasoning", - "encrypted_content": wrapped, - "summary": "", - "content": [], - }, + {"type": "reasoning", "encrypted_content": wrapped, "summary": "", "content": []}, ] ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) assert request_input == [ {"type": "reasoning", "summary": "plain string thought"}, - { - "type": "reasoning", - "content": [{"type": "output_text", "text": "in content"}], - }, + {"type": "reasoning", "content": [{"type": "output_text", "text": "in content"}]}, ] def test_leaves_input_untouched_when_no_encrypted_reasoning(self): request_input = [ {"role": "user", "content": "first turn"}, - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "no blob here"}], - }, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "no blob here"}]}, {"role": "user", "content": "second turn"}, ] before = [dict(item) for item in request_input] @@ -1877,9 +1724,7 @@ class TestStripEncryptedReasoningFromInput: def _cross_group_request_kwargs(): - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA-blob", "deployment-a" - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") return { "litellm_metadata": {}, "input": [ @@ -1935,9 +1780,7 @@ async def test_affinity_strips_encrypted_reasoning_when_routed_to_another_model_ assert result is routed_pool assert "_encrypted_content_affinity_pinned" not in request_kwargs - assert ( - request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True - ) + assert request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True assert request_kwargs["input"] is original_input assert [item.get("type") or item["role"] for item in original_input] == [ "user", @@ -1949,10 +1792,7 @@ async def test_affinity_strips_encrypted_reasoning_when_routed_to_another_model_ "type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}], } - assert not any( - isinstance(item, dict) and item.get("encrypted_content") - for item in original_input - ) + assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in original_input) @pytest.mark.asyncio @@ -1973,9 +1813,7 @@ async def test_affinity_fails_fast_within_the_origins_own_group(): "https://account-a.openai.azure.com/", "key-a", model_name="gpt-reasoning-tier" ) mock_router = _make_router_mock_with_cooldown( - originating, - cooldown_entries=[], - routed_group_model_ids=["deployment-a", "deployment-a2"], + originating, cooldown_entries=[], routed_group_model_ids=["deployment-a", "deployment-a2"] ) check = EncryptedContentAffinityCheck(router=mock_router) sibling_pool = [ @@ -2018,18 +1856,12 @@ async def test_affinity_does_not_strip_when_group_is_spelled_differently_but_sam EncryptedContentAffinityCheck, ) - originating = _make_originating_mock( - None, "key-a", model_name="openai/gpt-5.4-mini" - ) + originating = _make_originating_mock(None, "key-a", model_name="openai/gpt-5.4-mini") mock_router = _make_router_mock_with_cooldown( - originating, - cooldown_entries=[], - routed_group_model_ids=["deployment-mini-a", "deployment-mini-b"], + originating, cooldown_entries=[], routed_group_model_ids=["deployment-mini-a", "deployment-mini-b"] ) check = EncryptedContentAffinityCheck(router=mock_router) - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA-blob", "deployment-mini-a" - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-mini-a") sibling_pool = [ { "model_info": {"id": "deployment-mini-b"}, @@ -2080,18 +1912,12 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes( EncryptedContentAffinityCheck, ) - originating = _make_originating_mock( - None, "key-a", model_name="model_name_teamA_uuid" - ) + originating = _make_originating_mock(None, "key-a", model_name="model_name_teamA_uuid") mock_router = _make_router_mock_with_cooldown( - originating, - cooldown_entries=[], - routed_group_model_ids=["deployment-team-a", "deployment-team-b"], + originating, cooldown_entries=[], routed_group_model_ids=["deployment-team-a", "deployment-team-b"] ) check = EncryptedContentAffinityCheck(router=mock_router) - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - "gAAAAA-blob", "deployment-team-a" - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-team-a") sibling_pool = [ { "model_info": {"id": "deployment-team-b"}, @@ -2125,6 +1951,4 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes( ) assert request_kwargs["input"][1].get("encrypted_content") - mock_router.get_candidate_model_ids_for_route.assert_called_once_with( - model="team-public-model", team_id="teamA" - ) + mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA") diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index b0951b2b211..b4790132319 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2967,9 +2967,15 @@ def test_gemini_embedding_2_ga_in_cost_map(): assert info.get("mode") == "embedding" assert info.get("supports_multimodal") is True assert info.get("input_cost_per_token") == 2e-07 - assert info.get("input_cost_per_image") == 0.00012 - assert info.get("input_cost_per_audio_per_second") == 0.00016 - assert info.get("input_cost_per_video_per_second") == 0.00079 + if key.endswith("gemini-embedding-2"): + assert info.get("input_cost_per_audio_token") == 6.5e-06 + assert info.get("input_cost_per_image_token") == 4.5e-07 + assert info.get("input_cost_per_token_batches") == 1e-07 + assert info.get("input_cost_per_video_token") == 1.2e-05 + else: + assert info.get("input_cost_per_image") == 0.00012 + assert info.get("input_cost_per_audio_per_second") == 0.00016 + assert info.get("input_cost_per_video_per_second") == 0.00079 if provider in ("vertex_ai-embedding-models", "vertex_ai"): assert ( info.get("uses_embed_content") is True @@ -6503,3 +6509,4 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th assert snapshot["litellm_call_id"] assert snapshot["response_cost"] is not None assert snapshot["api_base"] +