mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(rate-limiter): count embedding data URIs as media in TPM estimates
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
71babd931b
commit
37f0b3f35d
3 changed files with 124 additions and 6 deletions
|
|
@ -386,6 +386,22 @@ def _tpm_input_media(value: object) -> _TPMMediaProjection:
|
|||
return _tpm_tool_result_media(value)
|
||||
|
||||
|
||||
def _tpm_embedding_media_leaf(value: object) -> _TPMMediaProjection:
|
||||
if isinstance(value, str) and value.startswith(("data:image/", "data:audio/")):
|
||||
projection: Final = _tpm_file_media({"type": "file", "file": {"file_data": value}})
|
||||
return _TPMMediaProjection("", projection.media)
|
||||
return _tpm_media_leaf(value)
|
||||
|
||||
|
||||
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
projection: Final = (
|
||||
_tpm_embedding_media_leaf(value)
|
||||
if isinstance(value, str) or as_str_mapping(value) is not None
|
||||
else _tpm_media_list(value, _tpm_embedding_media_leaf)
|
||||
)
|
||||
return projection.content, projection.media
|
||||
|
||||
|
||||
def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
projection: Final = _tpm_media_list(value, _tpm_input_media)
|
||||
return projection.content, projection.media
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
split_embedding_media_for_tpm,
|
||||
split_media_for_tpm,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
|
|
@ -1293,11 +1294,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
media_source: Final = messages if messages is not None else original_input
|
||||
projected, media = (
|
||||
split_media_for_tpm(media_source)
|
||||
split_embedding_media_for_tpm(original_input)
|
||||
if include_media and call_type in EMBEDDING_API_CALL_TYPES
|
||||
else split_media_for_tpm(media_source)
|
||||
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
|
||||
else (original_input, ())
|
||||
)
|
||||
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
|
||||
input_text: Final = (
|
||||
projected
|
||||
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
|
||||
else original_input
|
||||
)
|
||||
fallback_model: Final = request_model if isinstance(request_model, str) else None
|
||||
resolved_models: Final = (
|
||||
media_model_resolver(fallback_model or "", team_id)
|
||||
|
|
@ -3981,13 +3988,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
|
||||
)
|
||||
is_embedding_request: Final = self._is_embedding_request(data, call_type)
|
||||
embedding_text: Final = data.get("input") if is_embedding_request else None
|
||||
embedding_text: Final = (as_str_mapping(data) or {}).get("input") if is_embedding_request else None
|
||||
pretokenized_input_tokens: Final = (
|
||||
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
|
||||
)
|
||||
if pretokenized_input_tokens is not None:
|
||||
return pretokenized_input_tokens
|
||||
|
||||
embedding_projected, embedding_media = (
|
||||
split_embedding_media_for_tpm(embedding_text) if is_embedding_request else (embedding_text, ())
|
||||
)
|
||||
embedding_media_tokens: Final = sum(
|
||||
self._estimate_media_block_tokens(block, self._tokenizer_models_resolver(model or ""), model)
|
||||
for block in embedding_media
|
||||
)
|
||||
|
||||
prompt: Final = data.get("prompt")
|
||||
fallback_text: Final = prompt if prompt is not None else data.get("input")
|
||||
selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = (
|
||||
|
|
@ -4000,7 +4015,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
translated_request.get("tool_choice"),
|
||||
)
|
||||
if translated_request is not None
|
||||
else (None, embedding_text, data.get("tools"), data.get("tool_choice"))
|
||||
else (None, embedding_projected, data.get("tools"), data.get("tool_choice"))
|
||||
if is_embedding_request
|
||||
else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice"))
|
||||
if call_type in RERANK_API_CALL_TYPES
|
||||
|
|
@ -4032,12 +4047,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
for tokenizer_model in self._tokenizer_models_resolver(model or "")
|
||||
),
|
||||
)
|
||||
return estimate + audio_token_estimate
|
||||
return estimate + audio_token_estimate + embedding_media_tokens
|
||||
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
|
||||
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
|
||||
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
|
||||
data=data, call_type=call_type, include_media=False
|
||||
data=data, call_type=call_type, include_media=is_embedding_request
|
||||
)
|
||||
return estimated_input_tokens + audio_token_estimate
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,11 @@ Unit Tests for the max parallel request limiter v3 for the proxy
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextvars
|
||||
import io
|
||||
import logging
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Iterator, Sequence
|
||||
|
|
@ -18,6 +21,7 @@ import httpx
|
|||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
from PIL import Image
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -66,6 +70,89 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def embedding_image_uri() -> str:
|
||||
image: Final = Image.frombytes("RGB", (740, 560), random.Random(11217).randbytes(740 * 560 * 3))
|
||||
buffer: Final = io.BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=85)
|
||||
return "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ["string", "list", "file"])
|
||||
@pytest.mark.parametrize("precise", [False, True])
|
||||
def test_embedding_image_media_estimate(embedding_image_uri: str, shape: str, precise: bool) -> None:
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
value: Final = (
|
||||
embedding_image_uri
|
||||
if shape == "string"
|
||||
else [embedding_image_uri]
|
||||
if shape == "list"
|
||||
else [{"type": "file", "file": {"file_data": embedding_image_uri}}]
|
||||
)
|
||||
data: Final = {"model": "vertex_ai/gemini-embedding-2", "input": value}
|
||||
estimate: Final = (
|
||||
handler._estimate_precise_input_tokens(data, "vertex_ai/gemini-embedding-2", "aembedding")
|
||||
if precise
|
||||
else handler._estimate_input_and_output_tokens(data, call_type="aembedding")[0]
|
||||
)
|
||||
media_estimate: Final = handler._estimate_media_block_tokens(
|
||||
{"type": "image_url", "image_url": {"url": embedding_image_uri, "detail": "auto"}},
|
||||
("vertex_ai/gemini-embedding-2",),
|
||||
"vertex_ai/gemini-embedding-2",
|
||||
)
|
||||
assert media_estimate <= estimate < media_estimate + 100 < 75000 < len(embedding_image_uri) // 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "plain embedding text", ["plain", "text"], [1, 2, 3], [[1, 2], [3]]])
|
||||
def test_embedding_nonmedia_estimate_preserves_text_and_token_ids(value: object) -> None:
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
data: Final = {"model": "vertex_ai/gemini-embedding-2", "input": value}
|
||||
legacy: Final = handler._estimate_input_and_output_tokens(data, call_type="aembedding", include_media=False)
|
||||
assert handler._estimate_input_and_output_tokens(data, call_type="aembedding") == legacy
|
||||
token_count: Final = handler._count_pretokenized_embedding_input(value)
|
||||
expected: Final = (
|
||||
token_count
|
||||
if token_count is not None
|
||||
else litellm.token_counter(
|
||||
model="vertex_ai/gemini-embedding-2",
|
||||
text=value,
|
||||
)
|
||||
)
|
||||
assert handler._estimate_precise_input_tokens(data, "vertex_ai/gemini-embedding-2", "aembedding") == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("shape", ["string", "list", "file"])
|
||||
async def test_embedding_image_project_tpm_allows_media(embedding_image_uri: str, shape: str) -> None:
|
||||
cache: Final = DualCache()
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
|
||||
value: Final = (
|
||||
embedding_image_uri
|
||||
if shape == "string"
|
||||
else [embedding_image_uri]
|
||||
if shape == "list"
|
||||
else [{"type": "file", "file": {"file_data": embedding_image_uri}}]
|
||||
)
|
||||
caller: Final = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-embedding-media"),
|
||||
project_id="embedding-media",
|
||||
project_metadata={"model_tpm_limit": {"vertex_ai/gemini-embedding-2": 75000}},
|
||||
)
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=caller,
|
||||
cache=cache,
|
||||
data={"model": "vertex_ai/gemini-embedding-2", "input": value},
|
||||
call_type="aembedding",
|
||||
)
|
||||
reserved: Final = await cache.async_get_cache(
|
||||
key=handler.create_rate_limit_keys(
|
||||
"model_per_project", "embedding-media:vertex_ai/gemini-embedding-2", "tokens"
|
||||
),
|
||||
local_only=True,
|
||||
)
|
||||
assert 0 < reserved < 75000
|
||||
|
||||
|
||||
class TimeController:
|
||||
def __init__(self):
|
||||
self._current = datetime.utcnow()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue