mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge cc5ef3d424 into e16d79d635
This commit is contained in:
commit
b806d39ece
2 changed files with 135 additions and 4 deletions
|
|
@ -10,6 +10,7 @@ import itertools
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set
|
||||
from contextlib import asynccontextmanager
|
||||
|
|
@ -42,7 +43,11 @@ from litellm.caching.redis_batch import (
|
|||
active_request_redis_batch,
|
||||
)
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.constants import (
|
||||
DEFAULT_IMAGE_TOKEN_COUNT,
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -502,6 +507,11 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
|
|||
CallTypes.agenerate_content_stream.value,
|
||||
)
|
||||
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
|
||||
_IMAGE_CONTENT_TYPES: Final = frozenset({"input_image", "image_url", "image"})
|
||||
_DATA_IMAGE_PREFIX: Final = "data:image/"
|
||||
_DEFAULT_IMAGE_CHARS: Final[int] = int(DEFAULT_IMAGE_TOKEN_COUNT) * 4
|
||||
_LOW_DETAIL_IMAGE_CHARS: Final[int] = 85 * 4
|
||||
_BASE64_IMAGE_PATTERN: Final = re.compile(r"data:image/[^;'\"]+;base64,[A-Za-z0-9+/=]+")
|
||||
# litellm.token_counter has no per-type handling for "input_audio" content
|
||||
# blocks (unlike images, which use use_default_image_token_count) -- it
|
||||
# silently contributes 0 tokens for them. When the block carries a base64
|
||||
|
|
@ -1200,6 +1210,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
return max(round(normalized), 1)
|
||||
|
||||
@classmethod
|
||||
def _estimate_item_chars(cls, item: object) -> int:
|
||||
"""
|
||||
Estimate character count for an input item in Responses API or prompt endpoints.
|
||||
|
||||
Transport payloads containing multi-megabyte base64 images (such as in
|
||||
custom_tool_call_output.output or input_image blocks) are counted using
|
||||
bounded multimodal token estimates rather than measuring raw base64 text,
|
||||
preventing single requests from exhausting entire TPM quotas (#47814).
|
||||
"""
|
||||
s: Final[str] = str(item)
|
||||
if _DATA_IMAGE_PREFIX in s or ";base64," in s:
|
||||
target_image_chars: Final[int] = int(
|
||||
_LOW_DETAIL_IMAGE_CHARS
|
||||
if ('"detail": "low"' in s or "'detail': 'low'" in s)
|
||||
else _DEFAULT_IMAGE_CHARS
|
||||
)
|
||||
base64_chars: int = 0 # rebind-ok: accumulate image string reduction
|
||||
for match in _BASE64_IMAGE_PATTERN.finditer(s):
|
||||
m_len: int = len(match.group(0))
|
||||
if m_len > target_image_chars:
|
||||
base64_chars += m_len - target_image_chars
|
||||
return max(len(s) - base64_chars, target_image_chars)
|
||||
return len(s)
|
||||
|
||||
def _estimate_input_and_output_tokens(
|
||||
self,
|
||||
data: object,
|
||||
|
|
@ -1254,15 +1289,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
total_chars: Final = (
|
||||
len(get_str_from_messages(messages))
|
||||
self._estimate_item_chars(get_str_from_messages(messages))
|
||||
if isinstance(messages, list) and messages
|
||||
else len(prompt)
|
||||
if isinstance(prompt, str)
|
||||
else sum(len(str(item)) for item in prompt)
|
||||
if isinstance(prompt, list)
|
||||
else len(input_text)
|
||||
else self._estimate_item_chars(input_text)
|
||||
if isinstance(input_text, str)
|
||||
else sum(len(str(item)) for item in input_text)
|
||||
else sum(self._estimate_item_chars(item) for item in input_text)
|
||||
if isinstance(input_text, list)
|
||||
else 0
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3775,3 +3775,99 @@ def test_reservation_counts_a_compaction_item_by_its_summary(rate_limiter):
|
|||
) == handler._estimate_input_and_output_tokens(
|
||||
data={"model": "gpt-4o", "input": _COMPACTED_INPUT[2:], "max_output_tokens": 1}, call_type="aresponses"
|
||||
)
|
||||
|
||||
|
||||
def test_responses_api_custom_tool_output_image_base64_bounded(rate_limiter):
|
||||
"""
|
||||
Regression for #47814: Responses TPM reservation counts base64 in
|
||||
custom_tool_call_output images as text and doubles the estimate.
|
||||
Ensure structured images in custom_tool_call_output are bounded and do not inflate
|
||||
input or default output reservations into millions of tokens.
|
||||
"""
|
||||
handler, _cache = rate_limiter
|
||||
url = "data:image/png;base64," + "A" * 4_000_000
|
||||
data = {
|
||||
"model": "synthetic-test-model",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "Inspect the generated test images."}],
|
||||
},
|
||||
{
|
||||
"type": "custom_tool_call",
|
||||
"call_id": "call_synthetic",
|
||||
"name": "render_test",
|
||||
"input": "generate test images",
|
||||
},
|
||||
{
|
||||
"type": "custom_tool_call_output",
|
||||
"call_id": "call_synthetic",
|
||||
"output": [{"type": "input_text", "text": "Synthetic output only."}]
|
||||
+ [{"type": "input_image", "detail": "high", "image_url": url} for _ in range(3)],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
|
||||
data=data,
|
||||
call_type="aresponses",
|
||||
min_configured_tpm_limit=5_000_000,
|
||||
)
|
||||
|
||||
# 3 images should estimate ~750 tokens (250 tokens each) + text/tool metadata (<100 tokens), NOT 3,000,000+
|
||||
assert 750 <= input_tokens <= 1000
|
||||
assert output_tokens <= 2000
|
||||
assert input_tokens + output_tokens < 5_000_000
|
||||
|
||||
# With configured_output_tokens, operator override is respected
|
||||
i_cfg, o_cfg = handler._estimate_input_and_output_tokens(
|
||||
data=data,
|
||||
call_type="aresponses",
|
||||
min_configured_tpm_limit=5_000_000,
|
||||
configured_output_tokens=32768,
|
||||
)
|
||||
assert i_cfg == input_tokens
|
||||
assert o_cfg == 32768
|
||||
|
||||
|
||||
def test_responses_api_message_image_detail_low_and_high(rate_limiter):
|
||||
"""
|
||||
Verify detail='low' uses 85 tokens and detail='high' uses default image token count
|
||||
in Responses API message content while preserving adjacent text.
|
||||
"""
|
||||
handler, _cache = rate_limiter
|
||||
url = "data:image/png;base64," + "B" * 2_000_000
|
||||
data_low = {
|
||||
"model": "gpt-4o",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "A brief prompt"},
|
||||
{"type": "input_image", "detail": "low", "image_url": url},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
input_low, _ = handler._estimate_input_and_output_tokens(data=data_low, call_type="aresponses")
|
||||
# 85 tokens for low detail + text chars // 4
|
||||
assert 85 <= input_low <= 140
|
||||
|
||||
data_high = {
|
||||
"model": "gpt-4o",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "A brief prompt"},
|
||||
{"type": "input_image", "detail": "high", "image_url": url},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
input_high, _ = handler._estimate_input_and_output_tokens(data=data_high, call_type="aresponses")
|
||||
# 250 tokens for high detail + text chars // 4
|
||||
assert 250 <= input_high <= 300
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue