From ee173742a118089e6f40fc9d1542c755faa99352 Mon Sep 17 00:00:00 2001 From: VeeraBabu Jakkula Date: Thu, 22 Oct 2026 13:11:19 +0530 Subject: [PATCH 1/4] fix(proxy): use bounded multimodal token estimates for images in TPM reservations (#47814) --- .../hooks/parallel_request_limiter_v3.py | 76 ++++++++++++++- tests/unit/proxy/hooks/test_tpm_concurrent.py | 96 +++++++++++++++++++ 2 files changed, 167 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..60de0c94b18 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -42,7 +42,11 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DEFAULT_IMAGE_TOKEN_COUNT, + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -502,6 +506,10 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( CallTypes.agenerate_content_stream.value, ) RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16 +_IMAGE_CONTENT_TYPES: Final = frozenset({"input_image", "image_url", "image"}) +_DATA_IMAGE_PREFIX: Final = "data:image/" +_DEFAULT_IMAGE_CHARS: Final = DEFAULT_IMAGE_TOKEN_COUNT * DEFAULT_CHARS_PER_TOKEN +_LOW_DETAIL_IMAGE_CHARS: Final = 85 * DEFAULT_CHARS_PER_TOKEN # litellm.token_counter has no per-type handling for "input_audio" content # blocks (unlike images, which use use_default_image_token_count) -- it # silently contributes 0 tokens for them. When the block carries a base64 @@ -1200,6 +1208,64 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return max(round(normalized), 1) + @classmethod + def _count_message_image_chars(cls, messages: Sequence[object]) -> int: + """Count image content parts in chat completion messages using bounded char estimates.""" + count = 0 + for m in messages: + if not isinstance(m, Mapping): + continue + content = m.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_TYPES: + if part.get("detail") == "low": + count += _LOW_DETAIL_IMAGE_CHARS + else: + count += _DEFAULT_IMAGE_CHARS + return count + + @classmethod + def _estimate_structured_input_item_chars(cls, item: object, depth: int = 0) -> int: + """ + Estimate character count for an input item in Responses API or other endpoints. + + Structured images (such as input_image or image_url blocks nested in + message.content or custom_tool_call_output.output) and raw base64 data URLs + are counted using a bounded multimodal estimate (DEFAULT_IMAGE_TOKEN_COUNT) + rather than serializing raw transport payloads (which can be megabytes of base64 + characters) as text. Non-image text, tool definitions, tool arguments, and + call metadata remain fully preserved and counted. + """ + if item is None or depth > 10: + return 0 + if isinstance(item, str): + if item.startswith(_DATA_IMAGE_PREFIX) or ";base64," in item[:100]: + return _DEFAULT_IMAGE_CHARS + return len(item) + if isinstance(item, (int, float, bool)): + return len(str(item)) + if isinstance(item, (list, tuple, set)): + return sum(cls._estimate_structured_input_item_chars(x, depth + 1) for x in item) + if isinstance(item, Mapping): + item_type = item.get("type") + if item_type in _IMAGE_CONTENT_TYPES or ("image_url" in item and not isinstance(item.get("image_url"), list)): + if item.get("detail") == "low": + return _LOW_DETAIL_IMAGE_CHARS + return _DEFAULT_IMAGE_CHARS + total = 0 + for k, v in item.items(): + total += len(str(k)) + if isinstance(v, str) and (v.startswith(_DATA_IMAGE_PREFIX) or ";base64," in v[:100]): + total += _DEFAULT_IMAGE_CHARS + else: + total += cls._estimate_structured_input_item_chars(v, depth + 1) + return total + s = str(item) + if s.startswith(_DATA_IMAGE_PREFIX) or ";base64," in s[:100]: + return _DEFAULT_IMAGE_CHARS + return len(s) + def _estimate_input_and_output_tokens( self, data: object, @@ -1254,15 +1320,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) total_chars: Final = ( - len(get_str_from_messages(messages)) + (len(get_str_from_messages(messages)) + self._count_message_image_chars(messages)) if isinstance(messages, list) and messages else len(prompt) if isinstance(prompt, str) - else sum(len(str(item)) for item in prompt) + else sum(self._estimate_structured_input_item_chars(item) for item in prompt) if isinstance(prompt, list) - else len(input_text) + else self._estimate_structured_input_item_chars(input_text) if isinstance(input_text, str) - else sum(len(str(item)) for item in input_text) + else sum(self._estimate_structured_input_item_chars(item) for item in input_text) if isinstance(input_text, list) else 0 ) diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 3d1e494e36c..930f240c54e 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -3775,3 +3775,99 @@ def test_reservation_counts_a_compaction_item_by_its_summary(rate_limiter): ) == handler._estimate_input_and_output_tokens( data={"model": "gpt-4o", "input": _COMPACTED_INPUT[2:], "max_output_tokens": 1}, call_type="aresponses" ) + + +def test_responses_api_custom_tool_output_image_base64_bounded(rate_limiter): + """ + Regression for #47814: Responses TPM reservation counts base64 in + custom_tool_call_output images as text and doubles the estimate. + Ensure structured images in custom_tool_call_output are bounded and do not inflate + input or default output reservations into millions of tokens. + """ + handler, _cache = rate_limiter + url = "data:image/png;base64," + "A" * 4_000_000 + data = { + "model": "synthetic-test-model", + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Inspect the generated test images."}], + }, + { + "type": "custom_tool_call", + "call_id": "call_synthetic", + "name": "render_test", + "input": "generate test images", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_synthetic", + "output": [{"type": "input_text", "text": "Synthetic output only."}] + + [{"type": "input_image", "detail": "high", "image_url": url} for _ in range(3)], + }, + ], + } + + input_tokens, output_tokens = handler._estimate_input_and_output_tokens( + data=data, + call_type="aresponses", + min_configured_tpm_limit=5_000_000, + ) + + # 3 images should estimate ~750 tokens (250 tokens each) + text/tool metadata (<100 tokens), NOT 3,000,000+ + assert 750 <= input_tokens <= 1000 + assert output_tokens <= 2000 + assert input_tokens + output_tokens < 5_000_000 + + # With configured_output_tokens, operator override is respected + i_cfg, o_cfg = handler._estimate_input_and_output_tokens( + data=data, + call_type="aresponses", + min_configured_tpm_limit=5_000_000, + configured_output_tokens=32768, + ) + assert i_cfg == input_tokens + assert o_cfg == 32768 + + +def test_responses_api_message_image_detail_low_and_high(rate_limiter): + """ + Verify detail='low' uses 85 tokens and detail='high' uses default image token count + in Responses API message content while preserving adjacent text. + """ + handler, _cache = rate_limiter + url = "data:image/png;base64," + "B" * 2_000_000 + data_low = { + "model": "gpt-4o", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "A brief prompt"}, + {"type": "input_image", "detail": "low", "image_url": url}, + ], + } + ], + } + input_low, _ = handler._estimate_input_and_output_tokens(data=data_low, call_type="aresponses") + # 85 tokens for low detail + text chars // 4 + assert 85 <= input_low <= 120 + + data_high = { + "model": "gpt-4o", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "A brief prompt"}, + {"type": "input_image", "detail": "high", "image_url": url}, + ], + } + ], + } + input_high, _ = handler._estimate_input_and_output_tokens(data=data_high, call_type="aresponses") + # 250 tokens for high detail + text chars // 4 + assert 250 <= input_high <= 300 From b29f33174987117dc858d1f1dfb5d0fe6644a74b Mon Sep 17 00:00:00 2001 From: VeeraBabu Jakkula Date: Thu, 22 Oct 2026 13:32:59 +0530 Subject: [PATCH 2/4] style: format multi-line condition in structured image item estimation --- litellm/proxy/hooks/parallel_request_limiter_v3.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 60de0c94b18..74c1413a943 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -1249,7 +1249,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return sum(cls._estimate_structured_input_item_chars(x, depth + 1) for x in item) if isinstance(item, Mapping): item_type = item.get("type") - if item_type in _IMAGE_CONTENT_TYPES or ("image_url" in item and not isinstance(item.get("image_url"), list)): + if item_type in _IMAGE_CONTENT_TYPES or ( + "image_url" in item and not isinstance(item.get("image_url"), list) + ): if item.get("detail") == "low": return _LOW_DETAIL_IMAGE_CHARS return _DEFAULT_IMAGE_CHARS From 4b7628bad78e8d840abcc03237ee9c78664366c7 Mon Sep 17 00:00:00 2001 From: VeeraBabu Jakkula Date: Thu, 22 Oct 2026 13:55:06 +0530 Subject: [PATCH 3/4] style: satisfy type discipline gate on multimodal token estimation --- litellm/proxy/hooks/parallel_request_limiter_v3.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 74c1413a943..53d25825ac1 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -1211,11 +1211,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): @classmethod def _count_message_image_chars(cls, messages: Sequence[object]) -> int: """Count image content parts in chat completion messages using bounded char estimates.""" - count = 0 + count: int = 0 # rebind-ok: accumulate image chars across messages for m in messages: if not isinstance(m, Mapping): continue - content = m.get("content") + content: Final = m.get("content") if isinstance(content, list): for part in content: if isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_TYPES: @@ -1248,14 +1248,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if isinstance(item, (list, tuple, set)): return sum(cls._estimate_structured_input_item_chars(x, depth + 1) for x in item) if isinstance(item, Mapping): - item_type = item.get("type") + item_type: Final = item.get("type") if item_type in _IMAGE_CONTENT_TYPES or ( "image_url" in item and not isinstance(item.get("image_url"), list) ): if item.get("detail") == "low": return _LOW_DETAIL_IMAGE_CHARS return _DEFAULT_IMAGE_CHARS - total = 0 + total: int = 0 # rebind-ok: accumulate structured chars across fields for k, v in item.items(): total += len(str(k)) if isinstance(v, str) and (v.startswith(_DATA_IMAGE_PREFIX) or ";base64," in v[:100]): @@ -1263,7 +1263,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else: total += cls._estimate_structured_input_item_chars(v, depth + 1) return total - s = str(item) + s: Final = str(item) if s.startswith(_DATA_IMAGE_PREFIX) or ";base64," in s[:100]: return _DEFAULT_IMAGE_CHARS return len(s) From cc5ef3d42429349aceb923d697299bfaa807b0f7 Mon Sep 17 00:00:00 2001 From: VeeraBabu Jakkula Date: Thu, 22 Oct 2026 14:47:27 +0530 Subject: [PATCH 4/4] fix(proxy): refine multimodal base64 image bounds and type annotations --- .../hooks/parallel_request_limiter_v3.py | 87 ++++++------------- tests/unit/proxy/hooks/test_tpm_concurrent.py | 2 +- 2 files changed, 28 insertions(+), 61 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 53d25825ac1..5a5104a1844 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -10,6 +10,7 @@ import itertools import json import logging import os +import re import uuid from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set from contextlib import asynccontextmanager @@ -508,8 +509,9 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16 _IMAGE_CONTENT_TYPES: Final = frozenset({"input_image", "image_url", "image"}) _DATA_IMAGE_PREFIX: Final = "data:image/" -_DEFAULT_IMAGE_CHARS: Final = DEFAULT_IMAGE_TOKEN_COUNT * DEFAULT_CHARS_PER_TOKEN -_LOW_DETAIL_IMAGE_CHARS: Final = 85 * DEFAULT_CHARS_PER_TOKEN +_DEFAULT_IMAGE_CHARS: Final[int] = int(DEFAULT_IMAGE_TOKEN_COUNT) * 4 +_LOW_DETAIL_IMAGE_CHARS: Final[int] = 85 * 4 +_BASE64_IMAGE_PATTERN: Final = re.compile(r"data:image/[^;'\"]+;base64,[A-Za-z0-9+/=]+") # litellm.token_counter has no per-type handling for "input_audio" content # blocks (unlike images, which use use_default_image_token_count) -- it # silently contributes 0 tokens for them. When the block carries a base64 @@ -1209,63 +1211,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return max(round(normalized), 1) @classmethod - def _count_message_image_chars(cls, messages: Sequence[object]) -> int: - """Count image content parts in chat completion messages using bounded char estimates.""" - count: int = 0 # rebind-ok: accumulate image chars across messages - for m in messages: - if not isinstance(m, Mapping): - continue - content: Final = m.get("content") - if isinstance(content, list): - for part in content: - if isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_TYPES: - if part.get("detail") == "low": - count += _LOW_DETAIL_IMAGE_CHARS - else: - count += _DEFAULT_IMAGE_CHARS - return count - - @classmethod - def _estimate_structured_input_item_chars(cls, item: object, depth: int = 0) -> int: + def _estimate_item_chars(cls, item: object) -> int: """ - Estimate character count for an input item in Responses API or other endpoints. + Estimate character count for an input item in Responses API or prompt endpoints. - Structured images (such as input_image or image_url blocks nested in - message.content or custom_tool_call_output.output) and raw base64 data URLs - are counted using a bounded multimodal estimate (DEFAULT_IMAGE_TOKEN_COUNT) - rather than serializing raw transport payloads (which can be megabytes of base64 - characters) as text. Non-image text, tool definitions, tool arguments, and - call metadata remain fully preserved and counted. + Transport payloads containing multi-megabyte base64 images (such as in + custom_tool_call_output.output or input_image blocks) are counted using + bounded multimodal token estimates rather than measuring raw base64 text, + preventing single requests from exhausting entire TPM quotas (#47814). """ - if item is None or depth > 10: - return 0 - if isinstance(item, str): - if item.startswith(_DATA_IMAGE_PREFIX) or ";base64," in item[:100]: - return _DEFAULT_IMAGE_CHARS - return len(item) - if isinstance(item, (int, float, bool)): - return len(str(item)) - if isinstance(item, (list, tuple, set)): - return sum(cls._estimate_structured_input_item_chars(x, depth + 1) for x in item) - if isinstance(item, Mapping): - item_type: Final = item.get("type") - if item_type in _IMAGE_CONTENT_TYPES or ( - "image_url" in item and not isinstance(item.get("image_url"), list) - ): - if item.get("detail") == "low": - return _LOW_DETAIL_IMAGE_CHARS - return _DEFAULT_IMAGE_CHARS - total: int = 0 # rebind-ok: accumulate structured chars across fields - for k, v in item.items(): - total += len(str(k)) - if isinstance(v, str) and (v.startswith(_DATA_IMAGE_PREFIX) or ";base64," in v[:100]): - total += _DEFAULT_IMAGE_CHARS - else: - total += cls._estimate_structured_input_item_chars(v, depth + 1) - return total - s: Final = str(item) - if s.startswith(_DATA_IMAGE_PREFIX) or ";base64," in s[:100]: - return _DEFAULT_IMAGE_CHARS + s: Final[str] = str(item) + if _DATA_IMAGE_PREFIX in s or ";base64," in s: + target_image_chars: Final[int] = int( + _LOW_DETAIL_IMAGE_CHARS + if ('"detail": "low"' in s or "'detail': 'low'" in s) + else _DEFAULT_IMAGE_CHARS + ) + base64_chars: int = 0 # rebind-ok: accumulate image string reduction + for match in _BASE64_IMAGE_PATTERN.finditer(s): + m_len: int = len(match.group(0)) + if m_len > target_image_chars: + base64_chars += m_len - target_image_chars + return max(len(s) - base64_chars, target_image_chars) return len(s) def _estimate_input_and_output_tokens( @@ -1322,15 +1289,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) total_chars: Final = ( - (len(get_str_from_messages(messages)) + self._count_message_image_chars(messages)) + self._estimate_item_chars(get_str_from_messages(messages)) if isinstance(messages, list) and messages else len(prompt) if isinstance(prompt, str) - else sum(self._estimate_structured_input_item_chars(item) for item in prompt) + else sum(len(str(item)) for item in prompt) if isinstance(prompt, list) - else self._estimate_structured_input_item_chars(input_text) + else self._estimate_item_chars(input_text) if isinstance(input_text, str) - else sum(self._estimate_structured_input_item_chars(item) for item in input_text) + else sum(self._estimate_item_chars(item) for item in input_text) if isinstance(input_text, list) else 0 ) diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 930f240c54e..49547343361 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -3853,7 +3853,7 @@ def test_responses_api_message_image_detail_low_and_high(rate_limiter): } input_low, _ = handler._estimate_input_and_output_tokens(data=data_low, call_type="aresponses") # 85 tokens for low detail + text chars // 4 - assert 85 <= input_low <= 120 + assert 85 <= input_low <= 140 data_high = { "model": "gpt-4o",