diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 567b8452ae2..3332995429f 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -10,6 +10,7 @@ import itertools import json import logging import os +import re import uuid from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set from contextlib import asynccontextmanager @@ -42,7 +43,11 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DEFAULT_IMAGE_TOKEN_COUNT, + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -502,6 +507,11 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( CallTypes.agenerate_content_stream.value, ) RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16 +_IMAGE_CONTENT_TYPES: Final = frozenset({"input_image", "image_url", "image"}) +_DATA_IMAGE_PREFIX: Final = "data:image/" +_DEFAULT_IMAGE_CHARS: Final[int] = int(DEFAULT_IMAGE_TOKEN_COUNT) * 4 +_LOW_DETAIL_IMAGE_CHARS: Final[int] = 85 * 4 +_BASE64_IMAGE_PATTERN: Final = re.compile(r"data:image/[^;'\"]+;base64,[A-Za-z0-9+/=]+") # litellm.token_counter has no per-type handling for "input_audio" content # blocks (unlike images, which use use_default_image_token_count) -- it # silently contributes 0 tokens for them. When the block carries a base64 @@ -1200,6 +1210,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return max(round(normalized), 1) + @classmethod + def _estimate_item_chars(cls, item: object) -> int: + """ + Estimate character count for an input item in Responses API or prompt endpoints. + + Transport payloads containing multi-megabyte base64 images (such as in + custom_tool_call_output.output or input_image blocks) are counted using + bounded multimodal token estimates rather than measuring raw base64 text, + preventing single requests from exhausting entire TPM quotas (#47814). + """ + s: Final[str] = str(item) + if _DATA_IMAGE_PREFIX in s or ";base64," in s: + target_image_chars: Final[int] = int( + _LOW_DETAIL_IMAGE_CHARS + if ('"detail": "low"' in s or "'detail': 'low'" in s) + else _DEFAULT_IMAGE_CHARS + ) + base64_chars: int = 0 # rebind-ok: accumulate image string reduction + for match in _BASE64_IMAGE_PATTERN.finditer(s): + m_len: int = len(match.group(0)) + if m_len > target_image_chars: + base64_chars += m_len - target_image_chars + return max(len(s) - base64_chars, target_image_chars) + return len(s) + def _estimate_input_and_output_tokens( self, data: object, @@ -1254,15 +1289,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) total_chars: Final = ( - len(get_str_from_messages(messages)) + self._estimate_item_chars(get_str_from_messages(messages)) if isinstance(messages, list) and messages else len(prompt) if isinstance(prompt, str) else sum(len(str(item)) for item in prompt) if isinstance(prompt, list) - else len(input_text) + else self._estimate_item_chars(input_text) if isinstance(input_text, str) - else sum(len(str(item)) for item in input_text) + else sum(self._estimate_item_chars(item) for item in input_text) if isinstance(input_text, list) else 0 ) diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 3d1e494e36c..49547343361 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -3775,3 +3775,99 @@ def test_reservation_counts_a_compaction_item_by_its_summary(rate_limiter): ) == handler._estimate_input_and_output_tokens( data={"model": "gpt-4o", "input": _COMPACTED_INPUT[2:], "max_output_tokens": 1}, call_type="aresponses" ) + + +def test_responses_api_custom_tool_output_image_base64_bounded(rate_limiter): + """ + Regression for #47814: Responses TPM reservation counts base64 in + custom_tool_call_output images as text and doubles the estimate. + Ensure structured images in custom_tool_call_output are bounded and do not inflate + input or default output reservations into millions of tokens. + """ + handler, _cache = rate_limiter + url = "data:image/png;base64," + "A" * 4_000_000 + data = { + "model": "synthetic-test-model", + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Inspect the generated test images."}], + }, + { + "type": "custom_tool_call", + "call_id": "call_synthetic", + "name": "render_test", + "input": "generate test images", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_synthetic", + "output": [{"type": "input_text", "text": "Synthetic output only."}] + + [{"type": "input_image", "detail": "high", "image_url": url} for _ in range(3)], + }, + ], + } + + input_tokens, output_tokens = handler._estimate_input_and_output_tokens( + data=data, + call_type="aresponses", + min_configured_tpm_limit=5_000_000, + ) + + # 3 images should estimate ~750 tokens (250 tokens each) + text/tool metadata (<100 tokens), NOT 3,000,000+ + assert 750 <= input_tokens <= 1000 + assert output_tokens <= 2000 + assert input_tokens + output_tokens < 5_000_000 + + # With configured_output_tokens, operator override is respected + i_cfg, o_cfg = handler._estimate_input_and_output_tokens( + data=data, + call_type="aresponses", + min_configured_tpm_limit=5_000_000, + configured_output_tokens=32768, + ) + assert i_cfg == input_tokens + assert o_cfg == 32768 + + +def test_responses_api_message_image_detail_low_and_high(rate_limiter): + """ + Verify detail='low' uses 85 tokens and detail='high' uses default image token count + in Responses API message content while preserving adjacent text. + """ + handler, _cache = rate_limiter + url = "data:image/png;base64," + "B" * 2_000_000 + data_low = { + "model": "gpt-4o", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "A brief prompt"}, + {"type": "input_image", "detail": "low", "image_url": url}, + ], + } + ], + } + input_low, _ = handler._estimate_input_and_output_tokens(data=data_low, call_type="aresponses") + # 85 tokens for low detail + text chars // 4 + assert 85 <= input_low <= 140 + + data_high = { + "model": "gpt-4o", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "A brief prompt"}, + {"type": "input_image", "detail": "high", "image_url": url}, + ], + } + ], + } + input_high, _ = handler._estimate_input_and_output_tokens(data=data_high, call_type="aresponses") + # 250 tokens for high detail + text chars // 4 + assert 250 <= input_high <= 300