diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 90e79d4d853..42303cb738b 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,9 +3,11 @@ import base64 import io import math +import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, cast import anyio @@ -342,6 +344,117 @@ def calculate_img_tokens( return total_tokens +_IMAGE_HEADER_BASE64_CHARS: Final = 65_536 +_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType( + { + "gpt-4o": (85, 170), + "gpt-4.1": (85, 170), + "gpt-4o-mini": (2833, 5667), + "gpt-5": (70, 140), + "gpt-5.1": (70, 140), + "o1": (75, 150), + "o1-pro": (75, 150), + "o3": (75, 150), + } +) +_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType( + { + "gpt-6-astra": 1.2, + "gpt-5.6-sol": 1.2, + "gpt-5.6-terra": 1.2, + "gpt-5.6-luna": 1.2, + "gpt-5.5": 1.2, + "gpt-5.4": 1.2, + "gpt-5.4-mini": 1.2, + "gpt-5.4-nano": 1.2, + "gpt-5.2": 1.2, + "gpt-4.1-mini": 1.62, + } +) + + +def _inline_image_dimensions(data: str) -> tuple[int, int] | None: + separator: Final = data.find(",", 0, 256) + if not data.startswith("data:image/") or separator < 0 or not data[:separator].endswith(";base64"): + return None + encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS] + try: + header: Final = base64.b64decode(encoded_header, validate=True) + image_type: Final = get_image_type(header) + if image_type not in ("png", "gif", "jpeg", "webp"): + return None + if image_type == "webp" and header[12:16] not in (b"VP8X", b"VP8 ", b"VP8L"): + return None + width, height = get_image_dimensions(f"data:image/{image_type};base64,{encoded_header}") + except (ValueError, TypeError, struct.error): + return None + return (width, height) if width > 0 and height > 0 else None + + +def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None: + if model in ("gpt-5.2", "gpt-4.1-mini"): + return None if detail == "original" else (2048, 6144) + original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5") + effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail + if effective_detail == "low": + return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None) + if effective_detail == "high": + return (65_535 if model == "gpt-6-astra" else 2048, 2500) + return ( + (65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000) + ) + + +def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int: + dimension_scale: Final = min(1.0, max_dimension / max(width, height)) + fitted_width: Final = max(1, round(width * dimension_scale)) + fitted_height: Final = max(1, round(height * dimension_scale)) + patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32) + if budget is None or patches <= budget: + return patches + shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height)) + width_patches: Final = fitted_width * shrink / 32 + height_patches: Final = fitted_height * shrink / 32 + aligned_shrink: Final = shrink * min( + math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches + ) + resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink)) + resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink)) + return math.ceil(resized_width / 32) * math.ceil(resized_height / 32) + + +def estimate_image_tokens(model: str, data: str, detail: str = "auto") -> int: + """Estimate documented image policies from at most 48 KiB of inline headers, without fetching media.""" + if detail not in ("low", "high", "auto", "original"): + return DEFAULT_IMAGE_TOKEN_COUNT + provider_model: Final = re.sub(r"^(?:openai|azure|openrouter/openai)/", "", model) + family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model) + tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family) + multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family) + if (tile_costs is None and multiplier is None) or (tile_costs is not None and detail == "original"): + return DEFAULT_IMAGE_TOKEN_COUNT + if tile_costs is not None and detail == "low" and data.startswith(("https://", "http://")): + return tile_costs[0] + dimensions: Final = _inline_image_dimensions(data) + if dimensions is None: + return DEFAULT_IMAGE_TOKEN_COUNT + width, height = dimensions + if tile_costs is not None: + base_tokens, tile_tokens = tile_costs + if detail == "low": + return base_tokens + scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height)) + tiles: Final = math.ceil(max(1, math.floor(width * scale)) / 512) * math.ceil( + max(1, math.floor(height * scale)) / 512 + ) + return base_tokens + tiles * tile_tokens + limits: Final = _patch_image_limits(family, detail) + if limits is None or multiplier is None: + return DEFAULT_IMAGE_TOKEN_COUNT + max_dimension, patch_budget = limits + return math.ceil(_image_patch_count(width, height, max_dimension, patch_budget) * multiplier) + + TokenCounterFunction = Callable[[str], int] """ Type for a function that counts tokens in a string. diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 567b8452ae2..52d50738b7a 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -5,10 +5,12 @@ This is currently in development and not yet ready for production. """ import asyncio +import base64 import binascii import itertools import json import logging +import math import os import uuid from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set @@ -42,14 +44,19 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DEFAULT_IMAGE_TOKEN_COUNT, + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count from litellm.llms.azure.ptu_capacity import normalized_tokens from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( @@ -98,6 +105,7 @@ if TYPE_CHECKING: from opentelemetry.trace import Span as _Span from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache + from litellm.router import Router from litellm.types.agents import AgentResponse from litellm.types.caching import RedisPipelineIncrementOperation @@ -109,6 +117,7 @@ else: _REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object]) +_MESSAGE_CONTENT_OBJECTS: Final = TypeAdapter(list[Mapping[str, object]]) _MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None) @@ -135,6 +144,28 @@ class RateLimitedModel: return requested_limit if requested_limit is not None else limits.get(self.group) +@dataclass(frozen=True, slots=True) +class MediaTokenEstimate: + image_tokens: int = 0 + audio_tokens: int = 0 + + @property + def total(self) -> int: + return self.image_tokens + self.audio_tokens + + +def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]: + if router is None: + return (model,) + return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,) + + +def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]: + from litellm.proxy.proxy_server import llm_router + + return _resolve_media_models(llm_router, model, team_id) + + def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: from litellm.proxy.proxy_server import llm_router @@ -502,17 +533,8 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( CallTypes.agenerate_content_stream.value, ) RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16 -# litellm.token_counter has no per-type handling for "input_audio" content -# blocks (unlike images, which use use_default_image_token_count) -- it -# silently contributes 0 tokens for them. When the block carries a base64 -# payload, the estimate is derived from the decoded byte count; when the -# block is a reference without a payload (or the payload is missing), this -# flat per-block floor is used instead. DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300 -# Conservative bytes-per-token assumption for size-based audio estimation: -# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing -# the lowest reasonable bitrate means we never under-reserve for higher- -# quality audio recorded at the same wall-clock duration. +_AUDIO_TOKENS_PER_SECOND: Final = 10 _AUDIO_BYTES_PER_TOKEN: Final = 1600 # Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from # "model_per_project" (the combined-TPM descriptor) so both can be enforced @@ -867,11 +889,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ] = _resolve_ptu_team_ceiling_via_proxy_router, fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings, force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings, + media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now self._tag_rate_limit_resolver = tag_rate_limit_resolver self._model_group_resolver = model_group_resolver + self._media_model_resolver = media_model_resolver self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver @@ -1138,6 +1162,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit: int | None = None, call_type: str | None = None, configured_output_tokens: int | None = None, + media_estimate: MediaTokenEstimate | None = None, ) -> int: """ Estimate total tokens this request will consume so we can reserve them @@ -1164,6 +1189,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + media_estimate=media_estimate, ) total_estimated: Final = estimated_input_tokens + max_tokens_estimate @@ -1184,6 +1210,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type: str | None, configured_output_tokens: int | None, raw_estimate: int, + media_estimate: MediaTokenEstimate | None = None, ) -> int: """The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the output budget the way the ceiling does instead of the raw sum the other scopes reserve.""" @@ -1194,6 +1221,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + media_estimate=media_estimate, ) normalized: Final = normalized_tokens( ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate @@ -1206,6 +1234,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit: int | None = None, call_type: str | None = None, configured_output_tokens: int | None = None, + media_estimate: MediaTokenEstimate | None = None, ) -> tuple[int, int]: """ Estimate input tokens and output (max_tokens) budget separately, so @@ -1232,7 +1261,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if not isinstance(data, dict): return 0, 0 translated_data: Final = self._translate_google_genai_native_request(data, call_type) - estimable_data: Final = translated_data if translated_data is not None else data + estimable_data: Final = _REQUEST_RATE_LIMIT_DATA.validate_python( + translated_data if translated_data is not None else data + ) selected_fields: Final[tuple[object | None, object | None, object | None]] = ( (None, None, estimable_data.get("input")) if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES @@ -1253,6 +1284,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input ) + model_value: Final = estimable_data.get("model") + estimated_media_tokens: Final = ( + media_estimate + if media_estimate is not None + else self._estimate_message_media_tokens( + messages, + (model_value if isinstance(model_value, str) else "",), + ) + ).total total_chars: Final = ( len(get_str_from_messages(messages)) if isinstance(messages, list) and messages @@ -1267,7 +1307,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else 0 ) - estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + estimated_text_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + estimated_input_tokens: Final = estimated_text_tokens + estimated_media_tokens explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type) generates_no_output: Final = ( @@ -1283,7 +1324,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): max_tokens_estimate: Final = ( 0 if generates_no_output - or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None) + or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None) else explicit_max_tokens if explicit_max_tokens is not None else configured_output_tokens @@ -3752,27 +3793,77 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptor_key=descriptor_key, ) + @classmethod + def _estimate_message_media_tokens(cls, messages: object, models: tuple[str, ...] = ("",)) -> MediaTokenEstimate: + if not isinstance(messages, list): + return MediaTokenEstimate() + estimates: Final = tuple( + cls._estimate_content_media_tokens(message.get("content"), models) + for message in _MESSAGE_CONTENT_OBJECTS.validate_python(messages) + ) + return MediaTokenEstimate( + image_tokens=sum(estimate.image_tokens for estimate in estimates), + audio_tokens=sum(estimate.audio_tokens for estimate in estimates), + ) + + @classmethod + def _estimate_content_media_tokens(cls, content: object, models: tuple[str, ...]) -> MediaTokenEstimate: + if not isinstance(content, list): + return MediaTokenEstimate() + blocks: Final = _MESSAGE_CONTENT_OBJECTS.validate_python(content) + return MediaTokenEstimate( + image_tokens=sum( + cls._estimate_image_block_tokens(block, models) + for block in blocks + if block.get("type") in ("image_url", "image") + ), + audio_tokens=sum( + cls._estimate_audio_block_tokens(block) for block in blocks if block.get("type") == "input_audio" + ), + ) + + @staticmethod + def _estimate_image_block_tokens(block: Mapping[str, object], models: tuple[str, ...]) -> int: + image_url: Final = block.get("image_url") + source: Final = block.get("source") + image: Final = _callback_kwargs_mapping(image_url if image_url is not None else source) + source_data: Final = image.get("data") + url: Final = ( + image_url + if isinstance(image_url, str) + else image.get("url") + if image_url is not None or image.get("type") == "url" + else "data:image/unknown;base64," + source_data + if image.get("type") == "base64" and isinstance(source_data, str) + else "" + ) + detail: Final = image.get("detail") + return max( + ( + estimate_image_tokens( + model, + url if isinstance(url, str) else "", + detail if isinstance(detail, str) else "auto", + ) + for model in models + ), + default=DEFAULT_IMAGE_TOKEN_COUNT, + ) + @staticmethod def _estimate_audio_block_tokens(block: object) -> int: - """ - Token estimate for one ``input_audio`` content block. - - When the block carries a base64 ``data`` payload, the estimate comes - from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``), - assuming the lowest reasonable audio bitrate so we never under-reserve - for higher-quality recordings of the same duration. - - When no payload is present (reference-only block or missing ``data``), - falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``. - """ - if not isinstance(block, dict): + audio: Final = _callback_kwargs_mapping(_callback_kwargs_mapping(block).get("input_audio")) + b64_data: Final = audio.get("data") + if not isinstance(b64_data, str) or not b64_data: return DEFAULT_AUDIO_TOKEN_ESTIMATE - input_audio: Final = block.get("input_audio") - b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None - if b64_data and isinstance(b64_data, str): - decoded_bytes: Final = len(b64_data) * 3 // 4 - return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE) - return DEFAULT_AUDIO_TOKEN_ESTIMATE + try: + duration: Final = calculate_request_duration(base64.b64decode(b64_data, validate=True)) + if duration is not None and math.isfinite(duration) and duration > 0: + return max(1, math.ceil(duration * _AUDIO_TOKENS_PER_SECOND)) + except (ValueError, binascii.Error): + pass + decoded_bytes: Final = len(b64_data) * 3 // 4 + return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE) @classmethod def _estimate_audio_content_tokens(cls, messages: object) -> int: @@ -3799,7 +3890,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): Drop ``input_audio`` content blocks before passing ``messages`` to ``token_counter``, which raises ``ValueError`` on them (no per-type handling, unlike images). The audio contribution is added back - separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the + separately via the duration estimate or unreadable-audio fallback so the rest of the message (text/images/tools) still gets counted accurately instead of the whole call falling back to the cheap char-count estimate. """ @@ -3873,7 +3964,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): state: Final = data.get("state") return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}" - def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: + def _estimate_precise_input_tokens( + self, + data: object, + model: str | None, + call_type: str | None = None, + media_estimate: MediaTokenEstimate | None = None, + ) -> int: """ Model-aware input token estimate for the project ITPM reservation, using ``litellm.token_counter`` -- the same approach the @@ -3937,7 +4034,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) messages, selected_text, countable_tools, countable_tool_choice = selected_inputs - audio_token_estimate: Final = self._estimate_audio_content_tokens(messages) + audio_token_estimate: Final = ( + media_estimate.audio_tokens if media_estimate is not None else self._estimate_audio_content_tokens(messages) + ) countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages try: @@ -3958,7 +4057,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str): return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN) - estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type) + estimated_input_tokens, _ = self._estimate_input_and_output_tokens( + data={**data, "messages": countable_messages}, + call_type=call_type, + media_estimate=MediaTokenEstimate(image_tokens=media_estimate.image_tokens) + if media_estimate is not None + else None, + ) return estimated_input_tokens + audio_token_estimate async def _reserve_project_io_tokens_or_raise( @@ -3972,6 +4077,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type: str | None = None, endpoint_type: EndpointType = EndpointType.GENERIC, reuses_request_slot: bool = False, + media_estimate: MediaTokenEstimate | None = None, ) -> None: """ Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style @@ -4003,9 +4109,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): data=data, min_configured_tpm_limit=min_configured_otpm_limit, call_type=call_type, + media_estimate=media_estimate, ) raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=requested_model, call_type=call_type + data=data, + model=requested_model, + call_type=call_type, + media_estimate=media_estimate, ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( @@ -4323,6 +4433,35 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) has_tpm_limits: Final = bool(configured_tpm_limits) + needs_media_estimate: Final = (has_tpm_limits and self.tpm_reservation_enabled) or any( + d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for d in descriptors + ) + translated_request: Final = ( + self._translate_google_genai_native_request(data, call_type) if needs_media_estimate else None + ) + media_messages: Final = ( + None + if call_type + in RESPONSES_API_CALL_TYPES + + EMBEDDING_API_CALL_TYPES + + TEXT_COMPLETION_API_CALL_TYPES + + EXTRACTION_API_CALL_TYPES + else (translated_request if translated_request is not None else request_data).get("messages") + ) + media_estimate: Final = ( + await offload_token_count(self._estimate_message_media_tokens)( + _MESSAGE_CONTENT_OBJECTS.validate_python(media_messages), + self._media_model_resolver(requested_model or "", user_api_key_dict.team_id), + ) + if needs_media_estimate + and isinstance(media_messages, list) + and any( + isinstance(message.get("content"), list) + for message in _MESSAGE_CONTENT_OBJECTS.validate_python(media_messages) + ) + else None + ) + # Populated on a successful combined-TPM reservation below, so the # project ITPM/OTPM block further down can roll it back if a # different bucket in the same request subsequently hits its @@ -4366,6 +4505,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + media_estimate=media_estimate, ), 1, ) @@ -4390,6 +4530,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type=call_type, configured_output_tokens=configured_output_tokens, raw_estimate=estimated_tokens, + media_estimate=media_estimate, ) if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit: if not reuses_request_slot: @@ -4453,6 +4594,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): user_api_key_dict=user_api_key_dict, tpm_reservation_scopes=tpm_reservation_scopes, tpm_reservation_amount=tpm_reservation_amount, + media_estimate=media_estimate, call_type=call_type, endpoint_type=endpoint_type, reuses_request_slot=reuses_request_slot, diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..4943f3d063c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13795,7 +13795,9 @@ class Router: ] return named or self._get_wildcard_deployments(model_name=model_group) - def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]: + def resolved_litellm_models( + self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False + ) -> tuple[str, ...]: """The provider model strings `model_name` can actually be served by on this proxy. `get_model_list` composes every channel the request path itself uses (exact name, @@ -13808,6 +13810,11 @@ class Router: deployment behind it - so the fallback for an empty result is the caller's policy, never this function's. """ + if use_base_model: + return tuple( + self._cost_map_backend_model(Deployment.model_validate(deployment)) + for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () + ) return tuple( litellm_model for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 88af656ad2e..f91d2360469 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -35,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import ( _get_extrapolating_count_function, _get_tiktoken_count_function, calculate_img_tokens, + estimate_image_tokens, high_detail_image_token_upper_bound, offload_token_count, ) @@ -2025,6 +2026,111 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_ assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() +# Image policies verified 2026-10-24: https://developers.openai.com/api/docs/guides/images-vision +@pytest.mark.parametrize( + ("model", "width", "height", "detail", "expected"), + [ + ("gpt-4o", 16, 16, "high", 255), + ("gpt-4.1", 1024, 1024, "high", 765), + ("gpt-4o-mini", 1024, 1024, "high", 25501), + ("gpt-5.1", 1024, 1024, "auto", 630), + ("gpt-5", 1024, 1024, "low", 70), + ("o1", 1024, 1024, "high", 675), + ("o1-pro", 1024, 1024, "high", 675), + ("o3", 1024, 1024, "low", 75), + ("openai/gpt-4o-2024-08-06", 2048, 4096, "auto", 1105), + ("azure/gpt-4.1", 1024, 1024, "low", 85), + ("openrouter/openai/gpt-4o-mini", 16, 16, "low", 2833), + ("gpt-6-astra", 1024, 1024, "high", 1229), + ("gpt-6-astra", 2048, 2048, "high", 3000), + ("gpt-6-astra", 4096, 512, "high", 2458), + ("gpt-6-astra", 2048, 2048, "auto", 4916), + ("gpt-6-astra", 2048, 2048, "low", 308), + ("gpt-5.6-sol", 4096, 512, "high", 615), + ("gpt-5.6-terra", 2048, 2048, "original", 4916), + ("gpt-5.6-luna", 2048, 2048, "auto", 4916), + ("gpt-5.5", 6400, 6400, "original", 12000), + ("gpt-5.4", 2048, 2048, "low", 4916), + ("gpt-5.4-mini", 2048, 2048, "auto", 3000), + ("gpt-5.4-nano", 6400, 6400, "original", 12000), + ("gpt-5.2", 4096, 512, "auto", 615), + ("gpt-4.1-mini-2025-04-14", 1024, 1024, "low", 1659), + ("gpt-6-astra", 16, 16, "original", 2), + ], +) +def test_image_estimate_uses_model_resolution_and_detail( + model: str, width: int, height: int, detail: str, expected: int +) -> None: + assert estimate_image_tokens(model, _png_data_url(width, height), detail) == expected + + +@pytest.mark.parametrize( + ("model", "data", "detail"), + [ + ("unregistered-model", _png_data_url(1024, 1024), "high"), + ("gpt-4.1-unrecognized", _png_data_url(1024, 1024), "high"), + ("other/gpt-4o", _png_data_url(1024, 1024), "high"), + ("gpt-4.1-nano", _png_data_url(1024, 1024), "high"), + ("gpt-4o", _png_data_url(1024, 1024), "original"), + ("gpt-5.2", _png_data_url(1024, 1024), "original"), + ("gpt-6-astra", _png_data_url(1024, 1024), "invalid"), + ("gpt-4o", "", "high"), + ("gpt-4o", "data:image/png;base64,not base64!", "high"), + ("gpt-4o", "data:image/png;base64," + base64.b64encode(b"unknown format").decode(), "high"), + ("gpt-4o", "data:image/jpeg;base64,/9j/", "high"), + ("gpt-4o", "data:image/webp;base64," + base64.b64encode(b"RIFF\0\0\0\0WEBPbad!").decode(), "high"), + ("gpt-4o", _png_data_url(0, 1024), "high"), + ("gpt-4o", "data:image/png;base64," + base64.b64encode(b"\x89PNG\r\n\x1a\n").decode(), "high"), + ], +) +def test_image_estimate_falls_back_when_policy_or_dimensions_are_unavailable( + model: str, data: str, detail: str +) -> None: + assert estimate_image_tokens(model, data, detail) == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT + + +@pytest.mark.parametrize(("model", "detail", "expected"), [ + ("gpt-4o", "high", litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT), + ("gpt-6-astra", "original", litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT), + ("gpt-4o-mini", "low", 2833), +]) +def test_image_estimate_does_not_fetch_remote_media(model: str, detail: str, expected: int) -> None: + with patch("httpx.Client.send") as send: + assert estimate_image_tokens(model, "https://image.invalid/large.png", detail) == expected + send.assert_not_called() + + +def test_image_estimate_bounds_decoding_independent_of_payload_size() -> None: + header: Final = base64.b64decode(_png_data_url(1024, 1024).split(",", 1)[1]) + data: Final = "data:image/png;base64," + base64.b64encode(header + b"\0" * 4_000_000).decode() + with patch("base64.b64decode", wraps=base64.b64decode) as decode_header: + assert estimate_image_tokens("gpt-6-astra", data, "high") == 1229 + assert max(len(call.args[0]) for call in decode_header.call_args_list) <= 65_536 + + +@pytest.mark.parametrize("header", [ + b"GIF89a" + (1024).to_bytes(2, "little") + (512).to_bytes(2, "little"), + b"\xff\xd8\xff\xc0\x00\x0b\x08" + (512).to_bytes(2, "big") + (1024).to_bytes(2, "big"), + b"RIFF\0\0\0\0WEBPVP8X" + b"\0" * 8 + (1023).to_bytes(3, "little") + (511).to_bytes(3, "little"), + b"RIFF\0\0\0\0WEBPVP8 " + b"\0" * 10 + (1024).to_bytes(2, "little") + (512).to_bytes(2, "little"), + b"RIFF\0\0\0\0WEBPVP8L" + b"\0" * 5 + (1023 | (511 << 14)).to_bytes(4, "little"), +]) +def test_image_estimate_reads_supported_image_headers(header: bytes) -> None: + data: Final = "data:image/octet-stream;base64," + base64.b64encode(header).decode() + assert estimate_image_tokens("gpt-4o", data, "high") == estimate_image_tokens( + "gpt-4o", _png_data_url(1024, 512), "high" + ) + + +def test_image_estimate_falls_back_when_dimensions_are_beyond_header_budget() -> None: + jpeg: Final = ( + b"\xff\xd8\xff\xe1\xff\xff" + b"\0" * 65533 + + b"\xff\xc0\x00\x0b\x08" + (512).to_bytes(2, "big") + (1024).to_bytes(2, "big") + ) + data: Final = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode() + assert estimate_image_tokens("gpt-4o", data, "high") == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT + + HUB_SETUP_SCRIPT: Final = """ import json import sys diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 3d1e494e36c..7f7c4fb6516 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -15,19 +15,27 @@ Redis. """ import asyncio +import base64 +import io +import wave +from functools import partial import json from datetime import datetime, timedelta from typing import Any, Dict, Final import pytest +from litellm import Router from litellm.caching.caching import DualCache +from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, RateLimitedModel, _AUDIO_BYTES_PER_TOKEN, + _resolve_media_models, PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( @@ -53,6 +61,197 @@ def rate_limiter(): return handler, cache +@pytest.mark.asyncio +@pytest.mark.parametrize("media_type", ["image_url", "image", "input_audio"]) +async def test_combined_tpm_rejects_media_before_dispatch( + rate_limiter: tuple[RateLimitHandler, DualCache], media_type: str, +) -> None: + handler, cache = rate_limiter + media: Final = { + "type": media_type, + "image_url": {"url": "https://example.invalid/image.png", "detail": "high"}, + "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}, + "input_audio": {"data": "AAAA", "format": "wav"}, + } + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-media-tpm"), tpm_limit=100) + data: Final = { + "model": "unknown-deployment", + "messages": [{"role": "user", "content": [media]}], + "max_tokens": 16, + } + with pytest.raises(ProxyRateLimitError) as exc: + await handler.async_pre_call_hook( + user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion", + ) + assert exc.value.status_code == 429 + + +@pytest.mark.parametrize("output_cap", [None, 0, 16]) +@pytest.mark.parametrize("configured_output", [None, 32]) +def test_combined_tpm_media_is_bounded_and_not_contentless( + rate_limiter: tuple[RateLimitHandler, DualCache], + output_cap: int | None, configured_output: int | None, +) -> None: + handler, _ = rate_limiter + for image_data in ("AAAA", "A" * 4_000_000): + data: Final = { + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": { + "url": "data:image/png;base64," + image_data, + }}, + ]}], + "n": 2, + **({"max_tokens": output_cap} if output_cap is not None else {}), + } + input_tokens, output_tokens = handler._estimate_input_and_output_tokens( + data, call_type="acompletion", configured_output_tokens=configured_output, + ) + expected_output: Final = ( + output_cap if output_cap is not None else configured_output + if configured_output is not None else max( + DEFAULT_IMAGE_TOKEN_COUNT, handler.no_max_tokens_output_floor(None), + ) + ) + assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT + assert output_tokens == expected_output * 2 + + +def test_combined_tpm_preserves_text_and_counts_each_media_block_once( + rate_limiter: tuple[RateLimitHandler, DualCache], +) -> None: + handler, _ = rate_limiter + audio: Final = { + "type": "input_audio", "input_audio": {"data": "A" * 4_000_000, "format": "wav"}, + } + image: Final = {"type": "image_url", "image_url": "https://example.invalid/image.png"} + data: Final = { + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "abcd" * 10}, image, image, audio, + ]}], "max_tokens": 0, + } + assert handler._estimate_input_and_output_tokens(data) == ( + 10 + 2 * DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio), 0, + ) + + +def test_project_itpm_fallback_counts_chat_audio_once( + rate_limiter: tuple[RateLimitHandler, DualCache], +) -> None: + handler, _ = rate_limiter + audio: Final = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}} + data: Final = { + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "abcd" * 10}, audio, + {"type": "image_url", "image_url": {"url": "https://example.invalid/image.png"}}, + {"type": "unsupported_counter_block"}, + ]}], + } + assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == ( + 10 + DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio) + ) + + +def _audio_message(seconds: int, sample_rate: int, channels: int) -> dict[str, object]: + output: Final = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(channels) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(b"\x00\x00" * seconds * sample_rate * channels) + return {"role": "user", "content": [{ + "type": "input_audio", "input_audio": { + "data": base64.b64encode(output.getvalue()).decode(), "format": "wav", + }, + }]} + + +@pytest.mark.parametrize("sample_rate,channels", [(8000, 1), (24000, 1), (48000, 2)]) +def test_audio_estimates_follow_duration_instead_of_encoding_size( + rate_limiter: tuple[RateLimitHandler, DualCache], sample_rate: int, channels: int, +) -> None: + handler, _ = rate_limiter + # OpenAI user-audio estimate, checked 2026-10-24: + # https://developers.openai.com/api/docs/guides/voice-latency-cost?voice-api=realtime + for seconds in (1, 31): + message: Final = _audio_message(seconds, sample_rate, channels) + data: Final = {"model": "gpt-audio-1.5", "messages": [message], "max_tokens": 0} + assert handler._estimate_input_and_output_tokens(data) == (seconds * 10, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("seconds,admitted", [(1, True), (31, False)]) +async def test_audio_duration_drives_combined_and_project_admission( + rate_limiter: tuple[RateLimitHandler, DualCache], seconds: int, admitted: bool, +) -> None: + handler, cache = rate_limiter + auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-duration"), tpm_limit=100, project_id="audio-project", + project_metadata={"model_itpm_limit": {"gpt-audio-1.5": 100}}, + ) + data: Final = {"model": "gpt-audio-1.5", "messages": [_audio_message(seconds, 24000, 1)], "max_tokens": 16} + if not admitted: + with pytest.raises(ProxyRateLimitError): + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + return + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_tokens == 26 + assert 10 <= stash.itpm_reserved_tokens < 100 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("detail,admitted", [("low", True), ("high", False), ("auto", False)]) +@pytest.mark.parametrize("route", ["alias", "azure", "mixed"]) +async def test_model_specific_images_resolve_routes_before_admission( + detail: str, admitted: bool, route: str, +) -> None: + from PIL import Image + + output: Final = io.BytesIO() + Image.new("RGB", (1024, 1024)).save(output, format="PNG") + image: Final = "data:image/png;base64," + base64.b64encode(output.getvalue()).decode() + router: Final = Router( + model_list=[ + {"model_name": "vision", "litellm_params": {"model": "openai/gpt-5.1", "api_key": "test"}}, + {"model_name": "azure", "litellm_params": { + "model": "azure/custom-deployment", "base_model": "gpt-4o-mini", + "api_key": "test", "api_base": "https://example.invalid", "api_version": "2024-12-01-preview", + }, "model_info": {"base_model": "gpt-5.1"}}, + {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.1", "api_key": "test"}}, + {"model_name": "mixed", "litellm_params": {"model": "openai/gpt-4o", "api_key": "test"}}, + ], model_group_alias={"alias": "vision"}, + ) + cache: Final = DualCache() + handler: Final = RateLimitHandler( + internal_usage_cache=InternalUsageCache(cache), media_model_resolver=partial(_resolve_media_models, router), + ) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-image-policy"), tpm_limit=500) + data: Final = {"model": route, "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": image, "detail": detail}}, + ]}], "max_tokens": 16} + if not admitted: + with pytest.raises(ProxyRateLimitError): + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + return + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_tokens == (85 if route == "mixed" else 70) + 16 + + +@pytest.mark.parametrize("sample_rate,channels", [(8000, 1), (48000, 2)]) +def test_project_fallback_reuses_audio_duration_once( + rate_limiter: tuple[RateLimitHandler, DualCache], sample_rate: int, channels: int, +) -> None: + handler, _ = rate_limiter + data: Final = {"model": "unknown-deployment", "messages": [ + _audio_message(31, sample_rate, channels), + {"role": "user", "content": [{"type": "unsupported_counter_block"}]}, + ]} + assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == 310 + + @pytest.fixture(autouse=True) def _isolated_request_stash(): token = _request_stash.set(None) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 568a05e8ff4..42808c497b7 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -30149,3 +30149,21 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items(): assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False + + +@pytest.mark.parametrize("info_base,params_base", [(None, None), (None, "gpt-5.1"), ("gpt-6-astra", "gpt-5.1")]) +def test_resolved_litellm_models_can_use_configured_base_model( + info_base: str | None, params_base: str | None, +) -> None: + router: Final = Router( + model_list=[{ + "model_name": "deployment", "litellm_params": { + "model": "azure/custom-name", "api_key": "test", "api_base": "https://example.invalid", + "api_version": "2024-12-01-preview", **({"base_model": params_base} if params_base else {}), + }, "model_info": {"base_model": info_base}, + }], model_group_alias={"alias": "deployment"}, + ) + assert router.resolved_litellm_models("alias") == ("azure/custom-name",) + assert router.resolved_litellm_models("alias", use_base_model=True) == ( + info_base or params_base or "azure/custom-name", + )