fix(rate-limiter): count embedding data URIs as media in TPM estimates

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
gabriele 2026-10-29 11:51:13 +00:00
parent 71babd931b
commit 37f0b3f35d
3 changed files with 124 additions and 6 deletions

View file

@ -386,6 +386,22 @@ def _tpm_input_media(value: object) -> _TPMMediaProjection:
return _tpm_tool_result_media(value)
def _tpm_embedding_media_leaf(value: object) -> _TPMMediaProjection:
if isinstance(value, str) and value.startswith(("data:image/", "data:audio/")):
projection: Final = _tpm_file_media({"type": "file", "file": {"file_data": value}})
return _TPMMediaProjection("", projection.media)
return _tpm_media_leaf(value)
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = (
_tpm_embedding_media_leaf(value)
if isinstance(value, str) or as_str_mapping(value) is not None
else _tpm_media_list(value, _tpm_embedding_media_leaf)
)
return projection.content, projection.media
def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = _tpm_media_list(value, _tpm_input_media)
return projection.content, projection.media

View file

@ -49,6 +49,7 @@ from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
split_embedding_media_for_tpm,
split_media_for_tpm,
)
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
@ -1293,11 +1294,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
media_source: Final = messages if messages is not None else original_input
projected, media = (
split_media_for_tpm(media_source)
split_embedding_media_for_tpm(original_input)
if include_media and call_type in EMBEDDING_API_CALL_TYPES
else split_media_for_tpm(media_source)
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
else (original_input, ())
)
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
input_text: Final = (
projected
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
else original_input
)
fallback_model: Final = request_model if isinstance(request_model, str) else None
resolved_models: Final = (
media_model_resolver(fallback_model or "", team_id)
@ -3981,13 +3988,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
)
is_embedding_request: Final = self._is_embedding_request(data, call_type)
embedding_text: Final = data.get("input") if is_embedding_request else None
embedding_text: Final = (as_str_mapping(data) or {}).get("input") if is_embedding_request else None
pretokenized_input_tokens: Final = (
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
)
if pretokenized_input_tokens is not None:
return pretokenized_input_tokens
embedding_projected, embedding_media = (
split_embedding_media_for_tpm(embedding_text) if is_embedding_request else (embedding_text, ())
)
embedding_media_tokens: Final = sum(
self._estimate_media_block_tokens(block, self._tokenizer_models_resolver(model or ""), model)
for block in embedding_media
)
prompt: Final = data.get("prompt")
fallback_text: Final = prompt if prompt is not None else data.get("input")
selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = (
@ -4000,7 +4015,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
translated_request.get("tool_choice"),
)
if translated_request is not None
else (None, embedding_text, data.get("tools"), data.get("tool_choice"))
else (None, embedding_projected, data.get("tools"), data.get("tool_choice"))
if is_embedding_request
else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice"))
if call_type in RERANK_API_CALL_TYPES
@ -4032,12 +4047,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for tokenizer_model in self._tokenizer_models_resolver(model or "")
),
)
return estimate + audio_token_estimate
return estimate + audio_token_estimate + embedding_media_tokens
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
data=data, call_type=call_type, include_media=False
data=data, call_type=call_type, include_media=is_embedding_request
)
return estimated_input_tokens + audio_token_estimate

View file

@ -3,8 +3,11 @@ Unit Tests for the max parallel request limiter v3 for the proxy
"""
import asyncio
import base64
import contextvars
import io
import logging
import random
import sys
import time
from collections.abc import AsyncGenerator, Iterator, Sequence
@ -18,6 +21,7 @@ import httpx
import pytest
from fastapi import HTTPException
from pydantic import TypeAdapter
from PIL import Image
import litellm
from litellm import Router
@ -66,6 +70,89 @@ from litellm.types.utils import (
)
@pytest.fixture
def embedding_image_uri() -> str:
image: Final = Image.frombytes("RGB", (740, 560), random.Random(11217).randbytes(740 * 560 * 3))
buffer: Final = io.BytesIO()
image.save(buffer, format="JPEG", quality=85)
return "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode()
@pytest.mark.parametrize("shape", ["string", "list", "file"])
@pytest.mark.parametrize("precise", [False, True])
def test_embedding_image_media_estimate(embedding_image_uri: str, shape: str, precise: bool) -> None:
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
value: Final = (
embedding_image_uri
if shape == "string"
else [embedding_image_uri]
if shape == "list"
else [{"type": "file", "file": {"file_data": embedding_image_uri}}]
)
data: Final = {"model": "vertex_ai/gemini-embedding-2", "input": value}
estimate: Final = (
handler._estimate_precise_input_tokens(data, "vertex_ai/gemini-embedding-2", "aembedding")
if precise
else handler._estimate_input_and_output_tokens(data, call_type="aembedding")[0]
)
media_estimate: Final = handler._estimate_media_block_tokens(
{"type": "image_url", "image_url": {"url": embedding_image_uri, "detail": "auto"}},
("vertex_ai/gemini-embedding-2",),
"vertex_ai/gemini-embedding-2",
)
assert media_estimate <= estimate < media_estimate + 100 < 75000 < len(embedding_image_uri) // 4
@pytest.mark.parametrize("value", ["", "plain embedding text", ["plain", "text"], [1, 2, 3], [[1, 2], [3]]])
def test_embedding_nonmedia_estimate_preserves_text_and_token_ids(value: object) -> None:
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
data: Final = {"model": "vertex_ai/gemini-embedding-2", "input": value}
legacy: Final = handler._estimate_input_and_output_tokens(data, call_type="aembedding", include_media=False)
assert handler._estimate_input_and_output_tokens(data, call_type="aembedding") == legacy
token_count: Final = handler._count_pretokenized_embedding_input(value)
expected: Final = (
token_count
if token_count is not None
else litellm.token_counter(
model="vertex_ai/gemini-embedding-2",
text=value,
)
)
assert handler._estimate_precise_input_tokens(data, "vertex_ai/gemini-embedding-2", "aembedding") == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("shape", ["string", "list", "file"])
async def test_embedding_image_project_tpm_allows_media(embedding_image_uri: str, shape: str) -> None:
cache: Final = DualCache()
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
value: Final = (
embedding_image_uri
if shape == "string"
else [embedding_image_uri]
if shape == "list"
else [{"type": "file", "file": {"file_data": embedding_image_uri}}]
)
caller: Final = UserAPIKeyAuth(
api_key=hash_token("sk-embedding-media"),
project_id="embedding-media",
project_metadata={"model_tpm_limit": {"vertex_ai/gemini-embedding-2": 75000}},
)
await handler.async_pre_call_hook(
user_api_key_dict=caller,
cache=cache,
data={"model": "vertex_ai/gemini-embedding-2", "input": value},
call_type="aembedding",
)
reserved: Final = await cache.async_get_cache(
key=handler.create_rate_limit_keys(
"model_per_project", "embedding-media:vertex_ai/gemini-embedding-2", "tokens"
),
local_only=True,
)
assert 0 < reserved < 75000
class TimeController:
def __init__(self):
self._current = datetime.utcnow()