This commit is contained in:
VeeraBabu Jakkula 2026-10-27 02:24:38 +00:00 • committed by GitHub
commit b806d39ece
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 135 additions and 4 deletions

View file

@ -10,6 +10,7 @@ import itertools
import json
import logging
import os
import re
import uuid
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set
from contextlib import asynccontextmanager
@ -42,7 +43,11 @@ from litellm.caching.redis_batch import (
active_request_redis_batch,
)
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.constants import (
DEFAULT_IMAGE_TOKEN_COUNT,
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -502,6 +507,11 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
CallTypes.agenerate_content_stream.value,
)
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
_IMAGE_CONTENT_TYPES: Final = frozenset({"input_image", "image_url", "image"})
_DATA_IMAGE_PREFIX: Final = "data:image/"
_DEFAULT_IMAGE_CHARS: Final[int] = int(DEFAULT_IMAGE_TOKEN_COUNT) * 4
_LOW_DETAIL_IMAGE_CHARS: Final[int] = 85 * 4
_BASE64_IMAGE_PATTERN: Final = re.compile(r"data:image/[^;'\"]+;base64,[A-Za-z0-9+/=]+")
# litellm.token_counter has no per-type handling for "input_audio" content
# blocks (unlike images, which use use_default_image_token_count) -- it
# silently contributes 0 tokens for them. When the block carries a base64
@ -1200,6 +1210,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
return max(round(normalized), 1)
@classmethod
def _estimate_item_chars(cls, item: object) -> int:
"""
Estimate character count for an input item in Responses API or prompt endpoints.
Transport payloads containing multi-megabyte base64 images (such as in
custom_tool_call_output.output or input_image blocks) are counted using
bounded multimodal token estimates rather than measuring raw base64 text,
preventing single requests from exhausting entire TPM quotas (#47814).
"""
s: Final[str] = str(item)
if _DATA_IMAGE_PREFIX in s or ";base64," in s:
target_image_chars: Final[int] = int(
_LOW_DETAIL_IMAGE_CHARS
if ('"detail": "low"' in s or "'detail': 'low'" in s)
else _DEFAULT_IMAGE_CHARS
)
base64_chars: int = 0 # rebind-ok: accumulate image string reduction
for match in _BASE64_IMAGE_PATTERN.finditer(s):
m_len: int = len(match.group(0))
if m_len > target_image_chars:
base64_chars += m_len - target_image_chars
return max(len(s) - base64_chars, target_image_chars)
return len(s)
def _estimate_input_and_output_tokens(
self,
data: object,
@ -1254,15 +1289,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
total_chars: Final = (
len(get_str_from_messages(messages))
self._estimate_item_chars(get_str_from_messages(messages))
if isinstance(messages, list) and messages
else len(prompt)
if isinstance(prompt, str)
else sum(len(str(item)) for item in prompt)
if isinstance(prompt, list)
else len(input_text)
else self._estimate_item_chars(input_text)
if isinstance(input_text, str)
else sum(len(str(item)) for item in input_text)
else sum(self._estimate_item_chars(item) for item in input_text)
if isinstance(input_text, list)
else 0
)

View file

@ -3775,3 +3775,99 @@ def test_reservation_counts_a_compaction_item_by_its_summary(rate_limiter):
) == handler._estimate_input_and_output_tokens(
data={"model": "gpt-4o", "input": _COMPACTED_INPUT[2:], "max_output_tokens": 1}, call_type="aresponses"
)
def test_responses_api_custom_tool_output_image_base64_bounded(rate_limiter):
"""
Regression for #47814: Responses TPM reservation counts base64 in
custom_tool_call_output images as text and doubles the estimate.
Ensure structured images in custom_tool_call_output are bounded and do not inflate
input or default output reservations into millions of tokens.
"""
handler, _cache = rate_limiter
url = "data:image/png;base64," + "A" * 4_000_000
data = {
"model": "synthetic-test-model",
"input": [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Inspect the generated test images."}],
},
{
"type": "custom_tool_call",
"call_id": "call_synthetic",
"name": "render_test",
"input": "generate test images",
},
{
"type": "custom_tool_call_output",
"call_id": "call_synthetic",
"output": [{"type": "input_text", "text": "Synthetic output only."}]
+ [{"type": "input_image", "detail": "high", "image_url": url} for _ in range(3)],
},
],
}
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
data=data,
call_type="aresponses",
min_configured_tpm_limit=5_000_000,
)
# 3 images should estimate ~750 tokens (250 tokens each) + text/tool metadata (<100 tokens), NOT 3,000,000+
assert 750 <= input_tokens <= 1000
assert output_tokens <= 2000
assert input_tokens + output_tokens < 5_000_000
# With configured_output_tokens, operator override is respected
i_cfg, o_cfg = handler._estimate_input_and_output_tokens(
data=data,
call_type="aresponses",
min_configured_tpm_limit=5_000_000,
configured_output_tokens=32768,
)
assert i_cfg == input_tokens
assert o_cfg == 32768
def test_responses_api_message_image_detail_low_and_high(rate_limiter):
"""
Verify detail='low' uses 85 tokens and detail='high' uses default image token count
in Responses API message content while preserving adjacent text.
"""
handler, _cache = rate_limiter
url = "data:image/png;base64," + "B" * 2_000_000
data_low = {
"model": "gpt-4o",
"input": [
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "A brief prompt"},
{"type": "input_image", "detail": "low", "image_url": url},
],
}
],
}
input_low, _ = handler._estimate_input_and_output_tokens(data=data_low, call_type="aresponses")
# 85 tokens for low detail + text chars // 4
assert 85 <= input_low <= 140
data_high = {
"model": "gpt-4o",
"input": [
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "A brief prompt"},
{"type": "input_image", "detail": "high", "image_url": url},
],
}
],
}
input_high, _ = handler._estimate_input_and_output_tokens(data=data_high, call_type="aresponses")
# 250 tokens for high detail + text chars // 4
assert 250 <= input_high <= 300