This commit is contained in:
tin-berri 2026-10-26 01:28:38 -04:00 • committed by GitHub
commit 7dce87d487
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 624 additions and 39 deletions

View file

@ -3,9 +3,11 @@
import base64
import io
import math
import re
import struct
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
import anyio
@ -342,6 +344,117 @@ def calculate_img_tokens(
return total_tokens
_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
{
"gpt-4o": (85, 170),
"gpt-4.1": (85, 170),
"gpt-4o-mini": (2833, 5667),
"gpt-5": (70, 140),
"gpt-5.1": (70, 140),
"o1": (75, 150),
"o1-pro": (75, 150),
"o3": (75, 150),
}
)
_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
{
"gpt-6-astra": 1.2,
"gpt-5.6-sol": 1.2,
"gpt-5.6-terra": 1.2,
"gpt-5.6-luna": 1.2,
"gpt-5.5": 1.2,
"gpt-5.4": 1.2,
"gpt-5.4-mini": 1.2,
"gpt-5.4-nano": 1.2,
"gpt-5.2": 1.2,
"gpt-4.1-mini": 1.62,
}
)
def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
separator: Final = data.find(",", 0, 256)
if not data.startswith("data:image/") or separator < 0 or not data[:separator].endswith(";base64"):
return None
encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
try:
header: Final = base64.b64decode(encoded_header, validate=True)
image_type: Final = get_image_type(header)
if image_type not in ("png", "gif", "jpeg", "webp"):
return None
if image_type == "webp" and header[12:16] not in (b"VP8X", b"VP8 ", b"VP8L"):
return None
width, height = get_image_dimensions(f"data:image/{image_type};base64,{encoded_header}")
except (ValueError, TypeError, struct.error):
return None
return (width, height) if width > 0 and height > 0 else None
def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
if model in ("gpt-5.2", "gpt-4.1-mini"):
return None if detail == "original" else (2048, 6144)
original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
if effective_detail == "low":
return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
if effective_detail == "high":
return (65_535 if model == "gpt-6-astra" else 2048, 2500)
return (
(65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
)
def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
dimension_scale: Final = min(1.0, max_dimension / max(width, height))
fitted_width: Final = max(1, round(width * dimension_scale))
fitted_height: Final = max(1, round(height * dimension_scale))
patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
if budget is None or patches <= budget:
return patches
shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
width_patches: Final = fitted_width * shrink / 32
height_patches: Final = fitted_height * shrink / 32
aligned_shrink: Final = shrink * min(
math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
)
resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
def estimate_image_tokens(model: str, data: str, detail: str = "auto") -> int:
"""Estimate documented image policies from at most 48 KiB of inline headers, without fetching media."""
if detail not in ("low", "high", "auto", "original"):
return DEFAULT_IMAGE_TOKEN_COUNT
provider_model: Final = re.sub(r"^(?:openai|azure|openrouter/openai)/", "", model)
family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
if (tile_costs is None and multiplier is None) or (tile_costs is not None and detail == "original"):
return DEFAULT_IMAGE_TOKEN_COUNT
if tile_costs is not None and detail == "low" and data.startswith(("https://", "http://")):
return tile_costs[0]
dimensions: Final = _inline_image_dimensions(data)
if dimensions is None:
return DEFAULT_IMAGE_TOKEN_COUNT
width, height = dimensions
if tile_costs is not None:
base_tokens, tile_tokens = tile_costs
if detail == "low":
return base_tokens
scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
tiles: Final = math.ceil(max(1, math.floor(width * scale)) / 512) * math.ceil(
max(1, math.floor(height * scale)) / 512
)
return base_tokens + tiles * tile_tokens
limits: Final = _patch_image_limits(family, detail)
if limits is None or multiplier is None:
return DEFAULT_IMAGE_TOKEN_COUNT
max_dimension, patch_budget = limits
return math.ceil(_image_patch_count(width, height, max_dimension, patch_budget) * multiplier)
TokenCounterFunction = Callable[[str], int]
"""
Type for a function that counts tokens in a string.

View file

@ -5,10 +5,12 @@ This is currently in development and not yet ready for production.
"""
import asyncio
import base64
import binascii
import itertools
import json
import logging
import math
import os
import uuid
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set
@ -42,14 +44,19 @@ from litellm.caching.redis_batch import (
active_request_redis_batch,
)
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.constants import (
DEFAULT_IMAGE_TOKEN_COUNT,
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
from litellm.llms.azure.ptu_capacity import normalized_tokens
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
@ -98,6 +105,7 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.router import Router
from litellm.types.agents import AgentResponse
from litellm.types.caching import RedisPipelineIncrementOperation
@ -109,6 +117,7 @@ else:
_REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object])
_MESSAGE_CONTENT_OBJECTS: Final = TypeAdapter(list[Mapping[str, object]])
_MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None)
@ -135,6 +144,28 @@ class RateLimitedModel:
return requested_limit if requested_limit is not None else limits.get(self.group)
@dataclass(frozen=True, slots=True)
class MediaTokenEstimate:
image_tokens: int = 0
audio_tokens: int = 0
@property
def total(self) -> int:
return self.image_tokens + self.audio_tokens
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
if router is None:
return (model,)
return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,)
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
from litellm.proxy.proxy_server import llm_router
return _resolve_media_models(llm_router, model, team_id)
def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
from litellm.proxy.proxy_server import llm_router
@ -502,17 +533,8 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
CallTypes.agenerate_content_stream.value,
)
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
# litellm.token_counter has no per-type handling for "input_audio" content
# blocks (unlike images, which use use_default_image_token_count) -- it
# silently contributes 0 tokens for them. When the block carries a base64
# payload, the estimate is derived from the decoded byte count; when the
# block is a reference without a payload (or the payload is missing), this
# flat per-block floor is used instead.
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
# Conservative bytes-per-token assumption for size-based audio estimation:
# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing
# the lowest reasonable bitrate means we never under-reserve for higher-
# quality audio recorded at the same wall-clock duration.
_AUDIO_TOKENS_PER_SECOND: Final = 10
_AUDIO_BYTES_PER_TOKEN: Final = 1600
# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from
# "model_per_project" (the combined-TPM descriptor) so both can be enforced
@ -867,11 +889,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
] = _resolve_ptu_team_ceiling_via_proxy_router,
fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
):
self.internal_usage_cache = internal_usage_cache
self._time_provider = time_provider or datetime.now
self._tag_rate_limit_resolver = tag_rate_limit_resolver
self._model_group_resolver = model_group_resolver
self._media_model_resolver = media_model_resolver
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
self._fail_closed_resolver = fail_closed_resolver
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
@ -1138,6 +1162,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit: int | None = None,
call_type: str | None = None,
configured_output_tokens: int | None = None,
media_estimate: MediaTokenEstimate | None = None,
) -> int:
"""
Estimate total tokens this request will consume so we can reserve them
@ -1164,6 +1189,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
media_estimate=media_estimate,
)
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
@ -1184,6 +1210,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type: str | None,
configured_output_tokens: int | None,
raw_estimate: int,
media_estimate: MediaTokenEstimate | None = None,
) -> int:
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
@ -1194,6 +1221,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
media_estimate=media_estimate,
)
normalized: Final = normalized_tokens(
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
@ -1206,6 +1234,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit: int | None = None,
call_type: str | None = None,
configured_output_tokens: int | None = None,
media_estimate: MediaTokenEstimate | None = None,
) -> tuple[int, int]:
"""
Estimate input tokens and output (max_tokens) budget separately, so
@ -1232,7 +1261,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if not isinstance(data, dict):
return 0, 0
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
estimable_data: Final = translated_data if translated_data is not None else data
estimable_data: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(
translated_data if translated_data is not None else data
)
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
(None, None, estimable_data.get("input"))
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
@ -1253,6 +1284,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
)
model_value: Final = estimable_data.get("model")
estimated_media_tokens: Final = (
media_estimate
if media_estimate is not None
else self._estimate_message_media_tokens(
messages,
(model_value if isinstance(model_value, str) else "",),
)
).total
total_chars: Final = (
len(get_str_from_messages(messages))
if isinstance(messages, list) and messages
@ -1267,7 +1307,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else 0
)
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
estimated_text_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
estimated_input_tokens: Final = estimated_text_tokens + estimated_media_tokens
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
generates_no_output: Final = (
@ -1283,7 +1324,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
max_tokens_estimate: Final = (
0
if generates_no_output
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
else explicit_max_tokens
if explicit_max_tokens is not None
else configured_output_tokens
@ -3752,27 +3793,77 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
descriptor_key=descriptor_key,
)
@classmethod
def _estimate_message_media_tokens(cls, messages: object, models: tuple[str, ...] = ("",)) -> MediaTokenEstimate:
if not isinstance(messages, list):
return MediaTokenEstimate()
estimates: Final = tuple(
cls._estimate_content_media_tokens(message.get("content"), models)
for message in _MESSAGE_CONTENT_OBJECTS.validate_python(messages)
)
return MediaTokenEstimate(
image_tokens=sum(estimate.image_tokens for estimate in estimates),
audio_tokens=sum(estimate.audio_tokens for estimate in estimates),
)
@classmethod
def _estimate_content_media_tokens(cls, content: object, models: tuple[str, ...]) -> MediaTokenEstimate:
if not isinstance(content, list):
return MediaTokenEstimate()
blocks: Final = _MESSAGE_CONTENT_OBJECTS.validate_python(content)
return MediaTokenEstimate(
image_tokens=sum(
cls._estimate_image_block_tokens(block, models)
for block in blocks
if block.get("type") in ("image_url", "image")
),
audio_tokens=sum(
cls._estimate_audio_block_tokens(block) for block in blocks if block.get("type") == "input_audio"
),
)
@staticmethod
def _estimate_image_block_tokens(block: Mapping[str, object], models: tuple[str, ...]) -> int:
image_url: Final = block.get("image_url")
source: Final = block.get("source")
image: Final = _callback_kwargs_mapping(image_url if image_url is not None else source)
source_data: Final = image.get("data")
url: Final = (
image_url
if isinstance(image_url, str)
else image.get("url")
if image_url is not None or image.get("type") == "url"
else "data:image/unknown;base64," + source_data
if image.get("type") == "base64" and isinstance(source_data, str)
else ""
)
detail: Final = image.get("detail")
return max(
(
estimate_image_tokens(
model,
url if isinstance(url, str) else "",
detail if isinstance(detail, str) else "auto",
)
for model in models
),
default=DEFAULT_IMAGE_TOKEN_COUNT,
)
@staticmethod
def _estimate_audio_block_tokens(block: object) -> int:
"""
Token estimate for one ``input_audio`` content block.
When the block carries a base64 ``data`` payload, the estimate comes
from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``),
assuming the lowest reasonable audio bitrate so we never under-reserve
for higher-quality recordings of the same duration.
When no payload is present (reference-only block or missing ``data``),
falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``.
"""
if not isinstance(block, dict):
audio: Final = _callback_kwargs_mapping(_callback_kwargs_mapping(block).get("input_audio"))
b64_data: Final = audio.get("data")
if not isinstance(b64_data, str) or not b64_data:
return DEFAULT_AUDIO_TOKEN_ESTIMATE
input_audio: Final = block.get("input_audio")
b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
if b64_data and isinstance(b64_data, str):
decoded_bytes: Final = len(b64_data) * 3 // 4
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
return DEFAULT_AUDIO_TOKEN_ESTIMATE
try:
duration: Final = calculate_request_duration(base64.b64decode(b64_data, validate=True))
if duration is not None and math.isfinite(duration) and duration > 0:
return max(1, math.ceil(duration * _AUDIO_TOKENS_PER_SECOND))
except (ValueError, binascii.Error):
pass
decoded_bytes: Final = len(b64_data) * 3 // 4
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
@classmethod
def _estimate_audio_content_tokens(cls, messages: object) -> int:
@ -3799,7 +3890,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
Drop ``input_audio`` content blocks before passing ``messages`` to
``token_counter``, which raises ``ValueError`` on them (no per-type
handling, unlike images). The audio contribution is added back
separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the
separately via the duration estimate or unreadable-audio fallback so the rest of the
message (text/images/tools) still gets counted accurately instead of
the whole call falling back to the cheap char-count estimate.
"""
@ -3873,7 +3964,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
state: Final = data.get("state")
return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}"
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
def _estimate_precise_input_tokens(
self,
data: object,
model: str | None,
call_type: str | None = None,
media_estimate: MediaTokenEstimate | None = None,
) -> int:
"""
Model-aware input token estimate for the project ITPM reservation,
using ``litellm.token_counter`` -- the same approach the
@ -3937,7 +4034,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
messages, selected_text, countable_tools, countable_tool_choice = selected_inputs
audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
audio_token_estimate: Final = (
media_estimate.audio_tokens if media_estimate is not None else self._estimate_audio_content_tokens(messages)
)
countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
try:
@ -3958,7 +4057,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
data={**data, "messages": countable_messages},
call_type=call_type,
media_estimate=MediaTokenEstimate(image_tokens=media_estimate.image_tokens)
if media_estimate is not None
else None,
)
return estimated_input_tokens + audio_token_estimate
async def _reserve_project_io_tokens_or_raise(
@ -3972,6 +4077,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type: str | None = None,
endpoint_type: EndpointType = EndpointType.GENERIC,
reuses_request_slot: bool = False,
media_estimate: MediaTokenEstimate | None = None,
) -> None:
"""
Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style
@ -4003,9 +4109,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
data=data,
min_configured_tpm_limit=min_configured_otpm_limit,
call_type=call_type,
media_estimate=media_estimate,
)
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=requested_model, call_type=call_type
data=data,
model=requested_model,
call_type=call_type,
media_estimate=media_estimate,
)
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
estimated_output_tokens: Final = (
@ -4323,6 +4433,35 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
has_tpm_limits: Final = bool(configured_tpm_limits)
needs_media_estimate: Final = (has_tpm_limits and self.tpm_reservation_enabled) or any(
d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for d in descriptors
)
translated_request: Final = (
self._translate_google_genai_native_request(data, call_type) if needs_media_estimate else None
)
media_messages: Final = (
None
if call_type
in RESPONSES_API_CALL_TYPES
+ EMBEDDING_API_CALL_TYPES
+ TEXT_COMPLETION_API_CALL_TYPES
+ EXTRACTION_API_CALL_TYPES
else (translated_request if translated_request is not None else request_data).get("messages")
)
media_estimate: Final = (
await offload_token_count(self._estimate_message_media_tokens)(
_MESSAGE_CONTENT_OBJECTS.validate_python(media_messages),
self._media_model_resolver(requested_model or "", user_api_key_dict.team_id),
)
if needs_media_estimate
and isinstance(media_messages, list)
and any(
isinstance(message.get("content"), list)
for message in _MESSAGE_CONTENT_OBJECTS.validate_python(media_messages)
)
else None
)
# Populated on a successful combined-TPM reservation below, so the
# project ITPM/OTPM block further down can roll it back if a
# different bucket in the same request subsequently hits its
@ -4366,6 +4505,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
media_estimate=media_estimate,
),
1,
)
@ -4390,6 +4530,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type=call_type,
configured_output_tokens=configured_output_tokens,
raw_estimate=estimated_tokens,
media_estimate=media_estimate,
)
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
if not reuses_request_slot:
@ -4453,6 +4594,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
user_api_key_dict=user_api_key_dict,
tpm_reservation_scopes=tpm_reservation_scopes,
tpm_reservation_amount=tpm_reservation_amount,
media_estimate=media_estimate,
call_type=call_type,
endpoint_type=endpoint_type,
reuses_request_slot=reuses_request_slot,

View file

@ -13795,7 +13795,9 @@ class Router:
]
return named or self._get_wildcard_deployments(model_name=model_group)
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
def resolved_litellm_models(
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
) -> tuple[str, ...]:
"""The provider model strings `model_name` can actually be served by on this proxy.
`get_model_list` composes every channel the request path itself uses (exact name,
@ -13808,6 +13810,11 @@ class Router:
deployment behind it - so the fallback for an empty result is the caller's policy,
never this function's.
"""
if use_base_model:
return tuple(
self._cost_map_backend_model(Deployment.model_validate(deployment))
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
)
return tuple(
litellm_model
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()

View file

@ -35,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import (
_get_extrapolating_count_function,
_get_tiktoken_count_function,
calculate_img_tokens,
estimate_image_tokens,
high_detail_image_token_upper_bound,
offload_token_count,
)
@ -2025,6 +2026,111 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_
assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound()
# Image policies verified 2026-10-24: https://developers.openai.com/api/docs/guides/images-vision
@pytest.mark.parametrize(
("model", "width", "height", "detail", "expected"),
[
("gpt-4o", 16, 16, "high", 255),
("gpt-4.1", 1024, 1024, "high", 765),
("gpt-4o-mini", 1024, 1024, "high", 25501),
("gpt-5.1", 1024, 1024, "auto", 630),
("gpt-5", 1024, 1024, "low", 70),
("o1", 1024, 1024, "high", 675),
("o1-pro", 1024, 1024, "high", 675),
("o3", 1024, 1024, "low", 75),
("openai/gpt-4o-2024-08-06", 2048, 4096, "auto", 1105),
("azure/gpt-4.1", 1024, 1024, "low", 85),
("openrouter/openai/gpt-4o-mini", 16, 16, "low", 2833),
("gpt-6-astra", 1024, 1024, "high", 1229),
("gpt-6-astra", 2048, 2048, "high", 3000),
("gpt-6-astra", 4096, 512, "high", 2458),
("gpt-6-astra", 2048, 2048, "auto", 4916),
("gpt-6-astra", 2048, 2048, "low", 308),
("gpt-5.6-sol", 4096, 512, "high", 615),
("gpt-5.6-terra", 2048, 2048, "original", 4916),
("gpt-5.6-luna", 2048, 2048, "auto", 4916),
("gpt-5.5", 6400, 6400, "original", 12000),
("gpt-5.4", 2048, 2048, "low", 4916),
("gpt-5.4-mini", 2048, 2048, "auto", 3000),
("gpt-5.4-nano", 6400, 6400, "original", 12000),
("gpt-5.2", 4096, 512, "auto", 615),
("gpt-4.1-mini-2025-04-14", 1024, 1024, "low", 1659),
("gpt-6-astra", 16, 16, "original", 2),
],
)
def test_image_estimate_uses_model_resolution_and_detail(
model: str, width: int, height: int, detail: str, expected: int
) -> None:
assert estimate_image_tokens(model, _png_data_url(width, height), detail) == expected
@pytest.mark.parametrize(
("model", "data", "detail"),
[
("unregistered-model", _png_data_url(1024, 1024), "high"),
("gpt-4.1-unrecognized", _png_data_url(1024, 1024), "high"),
("other/gpt-4o", _png_data_url(1024, 1024), "high"),
("gpt-4.1-nano", _png_data_url(1024, 1024), "high"),
("gpt-4o", _png_data_url(1024, 1024), "original"),
("gpt-5.2", _png_data_url(1024, 1024), "original"),
("gpt-6-astra", _png_data_url(1024, 1024), "invalid"),
("gpt-4o", "", "high"),
("gpt-4o", "data:image/png;base64,not base64!", "high"),
("gpt-4o", "data:image/png;base64," + base64.b64encode(b"unknown format").decode(), "high"),
("gpt-4o", "data:image/jpeg;base64,/9j/", "high"),
("gpt-4o", "data:image/webp;base64," + base64.b64encode(b"RIFF\0\0\0\0WEBPbad!").decode(), "high"),
("gpt-4o", _png_data_url(0, 1024), "high"),
("gpt-4o", "data:image/png;base64," + base64.b64encode(b"\x89PNG\r\n\x1a\n").decode(), "high"),
],
)
def test_image_estimate_falls_back_when_policy_or_dimensions_are_unavailable(
model: str, data: str, detail: str
) -> None:
assert estimate_image_tokens(model, data, detail) == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
@pytest.mark.parametrize(("model", "detail", "expected"), [
("gpt-4o", "high", litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT),
("gpt-6-astra", "original", litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT),
("gpt-4o-mini", "low", 2833),
])
def test_image_estimate_does_not_fetch_remote_media(model: str, detail: str, expected: int) -> None:
with patch("httpx.Client.send") as send:
assert estimate_image_tokens(model, "https://image.invalid/large.png", detail) == expected
send.assert_not_called()
def test_image_estimate_bounds_decoding_independent_of_payload_size() -> None:
header: Final = base64.b64decode(_png_data_url(1024, 1024).split(",", 1)[1])
data: Final = "data:image/png;base64," + base64.b64encode(header + b"\0" * 4_000_000).decode()
with patch("base64.b64decode", wraps=base64.b64decode) as decode_header:
assert estimate_image_tokens("gpt-6-astra", data, "high") == 1229
assert max(len(call.args[0]) for call in decode_header.call_args_list) <= 65_536
@pytest.mark.parametrize("header", [
b"GIF89a" + (1024).to_bytes(2, "little") + (512).to_bytes(2, "little"),
b"\xff\xd8\xff\xc0\x00\x0b\x08" + (512).to_bytes(2, "big") + (1024).to_bytes(2, "big"),
b"RIFF\0\0\0\0WEBPVP8X" + b"\0" * 8 + (1023).to_bytes(3, "little") + (511).to_bytes(3, "little"),
b"RIFF\0\0\0\0WEBPVP8 " + b"\0" * 10 + (1024).to_bytes(2, "little") + (512).to_bytes(2, "little"),
b"RIFF\0\0\0\0WEBPVP8L" + b"\0" * 5 + (1023 | (511 << 14)).to_bytes(4, "little"),
])
def test_image_estimate_reads_supported_image_headers(header: bytes) -> None:
data: Final = "data:image/octet-stream;base64," + base64.b64encode(header).decode()
assert estimate_image_tokens("gpt-4o", data, "high") == estimate_image_tokens(
"gpt-4o", _png_data_url(1024, 512), "high"
)
def test_image_estimate_falls_back_when_dimensions_are_beyond_header_budget() -> None:
jpeg: Final = (
b"\xff\xd8\xff\xe1\xff\xff" + b"\0" * 65533
+ b"\xff\xc0\x00\x0b\x08" + (512).to_bytes(2, "big") + (1024).to_bytes(2, "big")
)
data: Final = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
assert estimate_image_tokens("gpt-4o", data, "high") == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
HUB_SETUP_SCRIPT: Final = """
import json
import sys

View file

@ -15,19 +15,27 @@ Redis.
"""
import asyncio
import base64
import io
import wave
from functools import partial
import json
from datetime import datetime, timedelta
from typing import Any, Dict, Final
import pytest
from litellm import Router
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROJECT_ITPM_DESCRIPTOR_KEY,
PROJECT_OTPM_DESCRIPTOR_KEY,
RateLimitedModel,
_AUDIO_BYTES_PER_TOKEN,
_resolve_media_models,
PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler,
)
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
@ -53,6 +61,197 @@ def rate_limiter():
return handler, cache
@pytest.mark.asyncio
@pytest.mark.parametrize("media_type", ["image_url", "image", "input_audio"])
async def test_combined_tpm_rejects_media_before_dispatch(
rate_limiter: tuple[RateLimitHandler, DualCache], media_type: str,
) -> None:
handler, cache = rate_limiter
media: Final = {
"type": media_type,
"image_url": {"url": "https://example.invalid/image.png", "detail": "high"},
"source": {"type": "base64", "media_type": "image/png", "data": "AAAA"},
"input_audio": {"data": "AAAA", "format": "wav"},
}
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-media-tpm"), tpm_limit=100)
data: Final = {
"model": "unknown-deployment",
"messages": [{"role": "user", "content": [media]}],
"max_tokens": 16,
}
with pytest.raises(ProxyRateLimitError) as exc:
await handler.async_pre_call_hook(
user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion",
)
assert exc.value.status_code == 429
@pytest.mark.parametrize("output_cap", [None, 0, 16])
@pytest.mark.parametrize("configured_output", [None, 32])
def test_combined_tpm_media_is_bounded_and_not_contentless(
rate_limiter: tuple[RateLimitHandler, DualCache],
output_cap: int | None, configured_output: int | None,
) -> None:
handler, _ = rate_limiter
for image_data in ("AAAA", "A" * 4_000_000):
data: Final = {
"messages": [{"role": "user", "content": [
{"type": "image_url", "image_url": {
"url": "data:image/png;base64," + image_data,
}},
]}],
"n": 2,
**({"max_tokens": output_cap} if output_cap is not None else {}),
}
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
data, call_type="acompletion", configured_output_tokens=configured_output,
)
expected_output: Final = (
output_cap if output_cap is not None else configured_output
if configured_output is not None else max(
DEFAULT_IMAGE_TOKEN_COUNT, handler.no_max_tokens_output_floor(None),
)
)
assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT
assert output_tokens == expected_output * 2
def test_combined_tpm_preserves_text_and_counts_each_media_block_once(
rate_limiter: tuple[RateLimitHandler, DualCache],
) -> None:
handler, _ = rate_limiter
audio: Final = {
"type": "input_audio", "input_audio": {"data": "A" * 4_000_000, "format": "wav"},
}
image: Final = {"type": "image_url", "image_url": "https://example.invalid/image.png"}
data: Final = {
"messages": [{"role": "user", "content": [
{"type": "text", "text": "abcd" * 10}, image, image, audio,
]}], "max_tokens": 0,
}
assert handler._estimate_input_and_output_tokens(data) == (
10 + 2 * DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio), 0,
)
def test_project_itpm_fallback_counts_chat_audio_once(
rate_limiter: tuple[RateLimitHandler, DualCache],
) -> None:
handler, _ = rate_limiter
audio: Final = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}
data: Final = {
"messages": [{"role": "user", "content": [
{"type": "text", "text": "abcd" * 10}, audio,
{"type": "image_url", "image_url": {"url": "https://example.invalid/image.png"}},
{"type": "unsupported_counter_block"},
]}],
}
assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == (
10 + DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio)
)
def _audio_message(seconds: int, sample_rate: int, channels: int) -> dict[str, object]:
output: Final = io.BytesIO()
with wave.open(output, "wb") as wav:
wav.setnchannels(channels)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes(b"\x00\x00" * seconds * sample_rate * channels)
return {"role": "user", "content": [{
"type": "input_audio", "input_audio": {
"data": base64.b64encode(output.getvalue()).decode(), "format": "wav",
},
}]}
@pytest.mark.parametrize("sample_rate,channels", [(8000, 1), (24000, 1), (48000, 2)])
def test_audio_estimates_follow_duration_instead_of_encoding_size(
rate_limiter: tuple[RateLimitHandler, DualCache], sample_rate: int, channels: int,
) -> None:
handler, _ = rate_limiter
# OpenAI user-audio estimate, checked 2026-10-24:
# https://developers.openai.com/api/docs/guides/voice-latency-cost?voice-api=realtime
for seconds in (1, 31):
message: Final = _audio_message(seconds, sample_rate, channels)
data: Final = {"model": "gpt-audio-1.5", "messages": [message], "max_tokens": 0}
assert handler._estimate_input_and_output_tokens(data) == (seconds * 10, 0)
@pytest.mark.asyncio
@pytest.mark.parametrize("seconds,admitted", [(1, True), (31, False)])
async def test_audio_duration_drives_combined_and_project_admission(
rate_limiter: tuple[RateLimitHandler, DualCache], seconds: int, admitted: bool,
) -> None:
handler, cache = rate_limiter
auth: Final = UserAPIKeyAuth(
api_key=hash_token("sk-duration"), tpm_limit=100, project_id="audio-project",
project_metadata={"model_itpm_limit": {"gpt-audio-1.5": 100}},
)
data: Final = {"model": "gpt-audio-1.5", "messages": [_audio_message(seconds, 24000, 1)], "max_tokens": 16}
if not admitted:
with pytest.raises(ProxyRateLimitError):
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
return
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
stash: Final = get_request_stash()
assert stash is not None
assert stash.reserved_tokens == 26
assert 10 <= stash.itpm_reserved_tokens < 100
@pytest.mark.asyncio
@pytest.mark.parametrize("detail,admitted", [("low", True), ("high", False), ("auto", False)])
@pytest.mark.parametrize("route", ["alias", "azure", "mixed"])
async def test_model_specific_images_resolve_routes_before_admission(
detail: str, admitted: bool, route: str,
) -> None:
from PIL import Image
output: Final = io.BytesIO()
Image.new("RGB", (1024, 1024)).save(output, format="PNG")
image: Final = "data:image/png;base64," + base64.b64encode(output.getvalue()).decode()
router: Final = Router(
model_list=[
{"model_name": "vision", "litellm_params": {"model": "openai/gpt-5.1", "api_key": "test"}},
{"model_name": "azure", "litellm_params": {
"model": "azure/custom-deployment", "base_model": "gpt-4o-mini",
"api_key": "test", "api_base": "https://example.invalid", "api_version": "2024-12-01-preview",
}, "model_info": {"base_model": "gpt-5.1"}},
{"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.1", "api_key": "test"}},
{"model_name": "mixed", "litellm_params": {"model": "openai/gpt-4o", "api_key": "test"}},
], model_group_alias={"alias": "vision"},
)
cache: Final = DualCache()
handler: Final = RateLimitHandler(
internal_usage_cache=InternalUsageCache(cache), media_model_resolver=partial(_resolve_media_models, router),
)
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-image-policy"), tpm_limit=500)
data: Final = {"model": route, "messages": [{"role": "user", "content": [
{"type": "image_url", "image_url": {"url": image, "detail": detail}},
]}], "max_tokens": 16}
if not admitted:
with pytest.raises(ProxyRateLimitError):
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
return
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
stash: Final = get_request_stash()
assert stash is not None
assert stash.reserved_tokens == (85 if route == "mixed" else 70) + 16
@pytest.mark.parametrize("sample_rate,channels", [(8000, 1), (48000, 2)])
def test_project_fallback_reuses_audio_duration_once(
rate_limiter: tuple[RateLimitHandler, DualCache], sample_rate: int, channels: int,
) -> None:
handler, _ = rate_limiter
data: Final = {"model": "unknown-deployment", "messages": [
_audio_message(31, sample_rate, channels),
{"role": "user", "content": [{"type": "unsupported_counter_block"}]},
]}
assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == 310
@pytest.fixture(autouse=True)
def _isolated_request_stash():
token = _request_stash.set(None)

View file

@ -30149,3 +30149,21 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items():
assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True
assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False
assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False
@pytest.mark.parametrize("info_base,params_base", [(None, None), (None, "gpt-5.1"), ("gpt-6-astra", "gpt-5.1")])
def test_resolved_litellm_models_can_use_configured_base_model(
info_base: str | None, params_base: str | None,
) -> None:
router: Final = Router(
model_list=[{
"model_name": "deployment", "litellm_params": {
"model": "azure/custom-name", "api_key": "test", "api_base": "https://example.invalid",
"api_version": "2024-12-01-preview", **({"base_model": params_base} if params_base else {}),
}, "model_info": {"base_model": info_base},
}], model_group_alias={"alias": "deployment"},
)
assert router.resolved_litellm_models("alias") == ("azure/custom-name",)
assert router.resolved_litellm_models("alias", use_base_model=True) == (
info_base or params_base or "azure/custom-name",
)