From 9316cf220c34230a578aac9bd61632f4759aecb3 Mon Sep 17 00:00:00 2001 From: Tin Date: Fri, 23 Oct 2026 16:59:48 -0400 Subject: [PATCH] fix(proxy): reserve TPM for image and audio messages --- .../hooks/parallel_request_limiter_v3.py | 43 ++++++++- tests/unit/proxy/hooks/test_tpm_concurrent.py | 92 +++++++++++++++++++ 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..0530270e8b6 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -42,7 +42,11 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DEFAULT_IMAGE_TOKEN_COUNT, + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -109,6 +113,7 @@ else: _REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object]) +_MESSAGE_CONTENT_OBJECTS: Final = TypeAdapter(list[Mapping[str, object]]) _MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None) @@ -1232,7 +1237,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if not isinstance(data, dict): return 0, 0 translated_data: Final = self._translate_google_genai_native_request(data, call_type) - estimable_data: Final = translated_data if translated_data is not None else data + estimable_data: Final = _REQUEST_RATE_LIMIT_DATA.validate_python( + translated_data if translated_data is not None else data + ) selected_fields: Final[tuple[object | None, object | None, object | None]] = ( (None, None, estimable_data.get("input")) if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES @@ -1253,6 +1260,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input ) + estimated_media_tokens: Final = self._estimate_message_media_tokens(messages) total_chars: Final = ( len(get_str_from_messages(messages)) if isinstance(messages, list) and messages @@ -1267,7 +1275,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else 0 ) - estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + estimated_text_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + estimated_input_tokens: Final = estimated_text_tokens + estimated_media_tokens explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type) generates_no_output: Final = ( @@ -1283,7 +1292,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): max_tokens_estimate: Final = ( 0 if generates_no_output - or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None) + or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None) else explicit_max_tokens if explicit_max_tokens is not None else configured_output_tokens @@ -3752,6 +3761,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptor_key=descriptor_key, ) + @classmethod + def _estimate_message_media_tokens(cls, messages: object) -> int: + if not isinstance(messages, list): + return 0 + return sum( + cls._estimate_content_media_tokens(message.get("content")) + for message in _MESSAGE_CONTENT_OBJECTS.validate_python(messages) + ) + + @classmethod + def _estimate_content_media_tokens(cls, content: object) -> int: + if not isinstance(content, list): + return 0 + return sum( + DEFAULT_IMAGE_TOKEN_COUNT + if block.get("type") in ("image_url", "image") + else cls._estimate_audio_block_tokens(block) + if block.get("type") == "input_audio" + else 0 + for block in _MESSAGE_CONTENT_OBJECTS.validate_python(content) + ) + @staticmethod def _estimate_audio_block_tokens(block: object) -> int: """ @@ -3958,7 +3989,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str): return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN) - estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type) + estimated_input_tokens, _ = self._estimate_input_and_output_tokens( + data={**data, "messages": countable_messages}, call_type=call_type + ) return estimated_input_tokens + audio_token_estimate async def _reserve_project_io_tokens_or_raise( diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 3d1e494e36c..6a95f84f241 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -22,7 +22,9 @@ from typing import Any, Dict, Final import pytest from litellm.caching.caching import DualCache +from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, @@ -53,6 +55,96 @@ def rate_limiter(): return handler, cache +@pytest.mark.asyncio +@pytest.mark.parametrize("media_type", ["image_url", "image", "input_audio"]) +async def test_combined_tpm_rejects_media_before_dispatch( + rate_limiter: tuple[RateLimitHandler, DualCache], media_type: str, +) -> None: + handler, cache = rate_limiter + media: Final = { + "type": media_type, + "image_url": {"url": "https://example.invalid/image.png", "detail": "high"}, + "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}, + "input_audio": {"data": "AAAA", "format": "wav"}, + } + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-media-tpm"), tpm_limit=100) + data: Final = { + "model": "unknown-deployment", + "messages": [{"role": "user", "content": [media]}], + "max_tokens": 16, + } + with pytest.raises(ProxyRateLimitError) as exc: + await handler.async_pre_call_hook( + user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion", + ) + assert exc.value.status_code == 429 + + +@pytest.mark.parametrize("output_cap", [None, 0, 16]) +@pytest.mark.parametrize("configured_output", [None, 32]) +def test_combined_tpm_media_is_bounded_and_not_contentless( + rate_limiter: tuple[RateLimitHandler, DualCache], + output_cap: int | None, configured_output: int | None, +) -> None: + handler, _ = rate_limiter + for image_data in ("AAAA", "A" * 4_000_000): + data: Final = { + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": { + "url": "data:image/png;base64," + image_data, + }}, + ]}], + "n": 2, + **({"max_tokens": output_cap} if output_cap is not None else {}), + } + input_tokens, output_tokens = handler._estimate_input_and_output_tokens( + data, call_type="acompletion", configured_output_tokens=configured_output, + ) + expected_output: Final = ( + output_cap if output_cap is not None else configured_output + if configured_output is not None else max( + DEFAULT_IMAGE_TOKEN_COUNT, handler.no_max_tokens_output_floor(None), + ) + ) + assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT + assert output_tokens == expected_output * 2 + + +def test_combined_tpm_preserves_text_and_counts_each_media_block_once( + rate_limiter: tuple[RateLimitHandler, DualCache], +) -> None: + handler, _ = rate_limiter + audio: Final = { + "type": "input_audio", "input_audio": {"data": "A" * 4_000_000, "format": "wav"}, + } + image: Final = {"type": "image_url", "image_url": "https://example.invalid/image.png"} + data: Final = { + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "abcd" * 10}, image, image, audio, + ]}], "max_tokens": 0, + } + assert handler._estimate_input_and_output_tokens(data) == ( + 10 + 2 * DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio), 0, + ) + + +def test_project_itpm_fallback_counts_chat_audio_once( + rate_limiter: tuple[RateLimitHandler, DualCache], +) -> None: + handler, _ = rate_limiter + audio: Final = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}} + data: Final = { + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "abcd" * 10}, audio, + {"type": "image_url", "image_url": {"url": "https://example.invalid/image.png"}}, + {"type": "unsupported_counter_block"}, + ]}], + } + assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == ( + 10 + DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio) + ) + + @pytest.fixture(autouse=True) def _isolated_request_stash(): token = _request_stash.set(None)