From 37f0b3f35d2f1452b01f07b35b41edbe736c8f0b Mon Sep 17 00:00:00 2001 From: gabriele Date: Thu, 29 Oct 2026 11:51:13 +0000 Subject: [PATCH] fix(rate-limiter): count embedding data URIs as media in TPM estimates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 16 ++++ .../hooks/parallel_request_limiter_v3.py | 27 ++++-- .../hooks/test_parallel_request_limiter_v3.py | 87 +++++++++++++++++++ 3 files changed, 124 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 9bc53791b36..625f50a2479 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -386,6 +386,22 @@ def _tpm_input_media(value: object) -> _TPMMediaProjection: return _tpm_tool_result_media(value) +def _tpm_embedding_media_leaf(value: object) -> _TPMMediaProjection: + if isinstance(value, str) and value.startswith(("data:image/", "data:audio/")): + projection: Final = _tpm_file_media({"type": "file", "file": {"file_data": value}}) + return _TPMMediaProjection("", projection.media) + return _tpm_media_leaf(value) + + +def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + projection: Final = ( + _tpm_embedding_media_leaf(value) + if isinstance(value, str) or as_str_mapping(value) is not None + else _tpm_media_list(value, _tpm_embedding_media_leaf) + ) + return projection.content, projection.media + + def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: projection: Final = _tpm_media_list(value, _tpm_input_media) return projection.content, projection.media diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a3a6f582dce..61af2d350d1 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -49,6 +49,7 @@ from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, + split_embedding_media_for_tpm, split_media_for_tpm, ) from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled @@ -1293,11 +1294,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): media_source: Final = messages if messages is not None else original_input projected, media = ( - split_media_for_tpm(media_source) + split_embedding_media_for_tpm(original_input) + if include_media and call_type in EMBEDDING_API_CALL_TYPES + else split_media_for_tpm(media_source) if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES) else (original_input, ()) ) - input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input + input_text: Final = ( + projected + if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES + else original_input + ) fallback_model: Final = request_model if isinstance(request_model, str) else None resolved_models: Final = ( media_model_resolver(fallback_model or "", team_id) @@ -3981,13 +3988,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): None if is_responses_request else self._translate_google_genai_native_request(data, call_type) ) is_embedding_request: Final = self._is_embedding_request(data, call_type) - embedding_text: Final = data.get("input") if is_embedding_request else None + embedding_text: Final = (as_str_mapping(data) or {}).get("input") if is_embedding_request else None pretokenized_input_tokens: Final = ( self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None ) if pretokenized_input_tokens is not None: return pretokenized_input_tokens + embedding_projected, embedding_media = ( + split_embedding_media_for_tpm(embedding_text) if is_embedding_request else (embedding_text, ()) + ) + embedding_media_tokens: Final = sum( + self._estimate_media_block_tokens(block, self._tokenizer_models_resolver(model or ""), model) + for block in embedding_media + ) + prompt: Final = data.get("prompt") fallback_text: Final = prompt if prompt is not None else data.get("input") selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = ( @@ -4000,7 +4015,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): translated_request.get("tool_choice"), ) if translated_request is not None - else (None, embedding_text, data.get("tools"), data.get("tool_choice")) + else (None, embedding_projected, data.get("tools"), data.get("tool_choice")) if is_embedding_request else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice")) if call_type in RERANK_API_CALL_TYPES @@ -4032,12 +4047,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): for tokenizer_model in self._tokenizer_models_resolver(model or "") ), ) - return estimate + audio_token_estimate + return estimate + audio_token_estimate + embedding_media_tokens except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str): return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN) estimated_input_tokens, _ = self._estimate_input_and_output_tokens( - data=data, call_type=call_type, include_media=False + data=data, call_type=call_type, include_media=is_embedding_request ) return estimated_input_tokens + audio_token_estimate diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index a38f1a54944..93d0c62f8a2 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3,8 +3,11 @@ Unit Tests for the max parallel request limiter v3 for the proxy """ import asyncio +import base64 import contextvars +import io import logging +import random import sys import time from collections.abc import AsyncGenerator, Iterator, Sequence @@ -18,6 +21,7 @@ import httpx import pytest from fastapi import HTTPException from pydantic import TypeAdapter +from PIL import Image import litellm from litellm import Router @@ -66,6 +70,89 @@ from litellm.types.utils import ( ) +@pytest.fixture +def embedding_image_uri() -> str: + image: Final = Image.frombytes("RGB", (740, 560), random.Random(11217).randbytes(740 * 560 * 3)) + buffer: Final = io.BytesIO() + image.save(buffer, format="JPEG", quality=85) + return "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode() + + +@pytest.mark.parametrize("shape", ["string", "list", "file"]) +@pytest.mark.parametrize("precise", [False, True]) +def test_embedding_image_media_estimate(embedding_image_uri: str, shape: str, precise: bool) -> None: + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + value: Final = ( + embedding_image_uri + if shape == "string" + else [embedding_image_uri] + if shape == "list" + else [{"type": "file", "file": {"file_data": embedding_image_uri}}] + ) + data: Final = {"model": "vertex_ai/gemini-embedding-2", "input": value} + estimate: Final = ( + handler._estimate_precise_input_tokens(data, "vertex_ai/gemini-embedding-2", "aembedding") + if precise + else handler._estimate_input_and_output_tokens(data, call_type="aembedding")[0] + ) + media_estimate: Final = handler._estimate_media_block_tokens( + {"type": "image_url", "image_url": {"url": embedding_image_uri, "detail": "auto"}}, + ("vertex_ai/gemini-embedding-2",), + "vertex_ai/gemini-embedding-2", + ) + assert media_estimate <= estimate < media_estimate + 100 < 75000 < len(embedding_image_uri) // 4 + + +@pytest.mark.parametrize("value", ["", "plain embedding text", ["plain", "text"], [1, 2, 3], [[1, 2], [3]]]) +def test_embedding_nonmedia_estimate_preserves_text_and_token_ids(value: object) -> None: + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + data: Final = {"model": "vertex_ai/gemini-embedding-2", "input": value} + legacy: Final = handler._estimate_input_and_output_tokens(data, call_type="aembedding", include_media=False) + assert handler._estimate_input_and_output_tokens(data, call_type="aembedding") == legacy + token_count: Final = handler._count_pretokenized_embedding_input(value) + expected: Final = ( + token_count + if token_count is not None + else litellm.token_counter( + model="vertex_ai/gemini-embedding-2", + text=value, + ) + ) + assert handler._estimate_precise_input_tokens(data, "vertex_ai/gemini-embedding-2", "aembedding") == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("shape", ["string", "list", "file"]) +async def test_embedding_image_project_tpm_allows_media(embedding_image_uri: str, shape: str) -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + value: Final = ( + embedding_image_uri + if shape == "string" + else [embedding_image_uri] + if shape == "list" + else [{"type": "file", "file": {"file_data": embedding_image_uri}}] + ) + caller: Final = UserAPIKeyAuth( + api_key=hash_token("sk-embedding-media"), + project_id="embedding-media", + project_metadata={"model_tpm_limit": {"vertex_ai/gemini-embedding-2": 75000}}, + ) + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=cache, + data={"model": "vertex_ai/gemini-embedding-2", "input": value}, + call_type="aembedding", + ) + reserved: Final = await cache.async_get_cache( + key=handler.create_rate_limit_keys( + "model_per_project", "embedding-media:vertex_ai/gemini-embedding-2", "tokens" + ), + local_only=True, + ) + assert 0 < reserved < 75000 + + class TimeController: def __init__(self): self._current = datetime.utcnow()