This commit is contained in:
tin-berri 2026-10-24 09:18:03 -07:00 • committed by GitHub
commit 39c77a6497
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 130 additions and 5 deletions

View file

@ -42,7 +42,11 @@ from litellm.caching.redis_batch import (
active_request_redis_batch,
)
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.constants import (
DEFAULT_IMAGE_TOKEN_COUNT,
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -109,6 +113,7 @@ else:
_REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object])
_MESSAGE_CONTENT_OBJECTS: Final = TypeAdapter(list[Mapping[str, object]])
_MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None)
@ -1232,7 +1237,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if not isinstance(data, dict):
return 0, 0
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
estimable_data: Final = translated_data if translated_data is not None else data
estimable_data: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(
translated_data if translated_data is not None else data
)
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
(None, None, estimable_data.get("input"))
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
@ -1253,6 +1260,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
)
estimated_media_tokens: Final = self._estimate_message_media_tokens(messages)
total_chars: Final = (
len(get_str_from_messages(messages))
if isinstance(messages, list) and messages
@ -1267,7 +1275,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else 0
)
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
estimated_text_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
estimated_input_tokens: Final = estimated_text_tokens + estimated_media_tokens
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
generates_no_output: Final = (
@ -1283,7 +1292,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
max_tokens_estimate: Final = (
0
if generates_no_output
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
else explicit_max_tokens
if explicit_max_tokens is not None
else configured_output_tokens
@ -3752,6 +3761,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
descriptor_key=descriptor_key,
)
@classmethod
def _estimate_message_media_tokens(cls, messages: object) -> int:
if not isinstance(messages, list):
return 0
return sum(
cls._estimate_content_media_tokens(message.get("content"))
for message in _MESSAGE_CONTENT_OBJECTS.validate_python(messages)
)
@classmethod
def _estimate_content_media_tokens(cls, content: object) -> int:
if not isinstance(content, list):
return 0
return sum(
DEFAULT_IMAGE_TOKEN_COUNT
if block.get("type") in ("image_url", "image")
else cls._estimate_audio_block_tokens(block)
if block.get("type") == "input_audio"
else 0
for block in _MESSAGE_CONTENT_OBJECTS.validate_python(content)
)
@staticmethod
def _estimate_audio_block_tokens(block: object) -> int:
"""
@ -3958,7 +3989,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
data={**data, "messages": countable_messages}, call_type=call_type
)
return estimated_input_tokens + audio_token_estimate
async def _reserve_project_io_tokens_or_raise(

View file

@ -22,7 +22,9 @@ from typing import Any, Dict, Final
import pytest
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROJECT_ITPM_DESCRIPTOR_KEY,
PROJECT_OTPM_DESCRIPTOR_KEY,
@ -53,6 +55,96 @@ def rate_limiter():
return handler, cache
@pytest.mark.asyncio
@pytest.mark.parametrize("media_type", ["image_url", "image", "input_audio"])
async def test_combined_tpm_rejects_media_before_dispatch(
rate_limiter: tuple[RateLimitHandler, DualCache], media_type: str,
) -> None:
handler, cache = rate_limiter
media: Final = {
"type": media_type,
"image_url": {"url": "https://example.invalid/image.png", "detail": "high"},
"source": {"type": "base64", "media_type": "image/png", "data": "AAAA"},
"input_audio": {"data": "AAAA", "format": "wav"},
}
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-media-tpm"), tpm_limit=100)
data: Final = {
"model": "unknown-deployment",
"messages": [{"role": "user", "content": [media]}],
"max_tokens": 16,
}
with pytest.raises(ProxyRateLimitError) as exc:
await handler.async_pre_call_hook(
user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion",
)
assert exc.value.status_code == 429
@pytest.mark.parametrize("output_cap", [None, 0, 16])
@pytest.mark.parametrize("configured_output", [None, 32])
def test_combined_tpm_media_is_bounded_and_not_contentless(
rate_limiter: tuple[RateLimitHandler, DualCache],
output_cap: int | None, configured_output: int | None,
) -> None:
handler, _ = rate_limiter
for image_data in ("AAAA", "A" * 4_000_000):
data: Final = {
"messages": [{"role": "user", "content": [
{"type": "image_url", "image_url": {
"url": "data:image/png;base64," + image_data,
}},
]}],
"n": 2,
**({"max_tokens": output_cap} if output_cap is not None else {}),
}
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
data, call_type="acompletion", configured_output_tokens=configured_output,
)
expected_output: Final = (
output_cap if output_cap is not None else configured_output
if configured_output is not None else max(
DEFAULT_IMAGE_TOKEN_COUNT, handler.no_max_tokens_output_floor(None),
)
)
assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT
assert output_tokens == expected_output * 2
def test_combined_tpm_preserves_text_and_counts_each_media_block_once(
rate_limiter: tuple[RateLimitHandler, DualCache],
) -> None:
handler, _ = rate_limiter
audio: Final = {
"type": "input_audio", "input_audio": {"data": "A" * 4_000_000, "format": "wav"},
}
image: Final = {"type": "image_url", "image_url": "https://example.invalid/image.png"}
data: Final = {
"messages": [{"role": "user", "content": [
{"type": "text", "text": "abcd" * 10}, image, image, audio,
]}], "max_tokens": 0,
}
assert handler._estimate_input_and_output_tokens(data) == (
10 + 2 * DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio), 0,
)
def test_project_itpm_fallback_counts_chat_audio_once(
rate_limiter: tuple[RateLimitHandler, DualCache],
) -> None:
handler, _ = rate_limiter
audio: Final = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}
data: Final = {
"messages": [{"role": "user", "content": [
{"type": "text", "text": "abcd" * 10}, audio,
{"type": "image_url", "image_url": {"url": "https://example.invalid/image.png"}},
{"type": "unsupported_counter_block"},
]}],
}
assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == (
10 + DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio)
)
@pytest.fixture(autouse=True)
def _isolated_request_stash():
token = _request_stash.set(None)