diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index dab3e48f91a..ded24cb0e3d 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -2,8 +2,12 @@ Utils used for litellm.transcription() and litellm.atranscription() """ +import base64 +import binascii import hashlib +import math import os +from collections.abc import Sequence from dataclasses import dataclass from typing import Final @@ -331,6 +335,44 @@ def calculate_request_duration(file: FileTypes) -> float | None: return None +DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300 +AUDIO_TOKENS_PER_SECOND: Final = 10 +AUDIO_BYTES_PER_TOKEN: Final = 1600 +_MAX_AUDIO_ESTIMATE_BYTES: Final = 16 * 1024 * 1024 + + +def _audio_token_rate(model: str) -> int | None: + if model.startswith( + ("gemini-", "gemini/gemini-", "vertex_ai/gemini-", "vertex_ai_beta/gemini-", "openrouter/google/gemini-") + ): + return 32 + if model.startswith(("gpt-", "openai/gpt-", "azure/gpt-", "openrouter/openai/gpt-")): + return AUDIO_TOKENS_PER_SECOND + return None + + +def estimate_audio_tokens(data: str | None, models: Sequence[str] = (), fallback_model: str | None = None) -> int: + """Estimate audio tokens from container duration within a bounded decoding budget.""" + if not data: + return DEFAULT_AUDIO_TOKEN_ESTIMATE + fallback_estimate: Final = max(len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE) + padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0 + if len(data) * 3 // 4 - padding > _MAX_AUDIO_ESTIMATE_BYTES: + return fallback_estimate + try: + duration: Final = calculate_request_duration(base64.b64decode(data, validate=True)) + except (ValueError, binascii.Error): + return fallback_estimate + if duration is None or not math.isfinite(duration) or duration <= 0: + return fallback_estimate + fallback_rate: Final = _audio_token_rate(fallback_model) if fallback_model is not None else None + rate: Final = max( + (_audio_token_rate(model) or fallback_rate or AUDIO_TOKENS_PER_SECOND for model in models), + default=fallback_rate or AUDIO_TOKENS_PER_SECOND, + ) + return max(1, math.ceil(duration * rate)) + + DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg" diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a73e50c8e9a..5b8e4d583bc 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -9,6 +9,7 @@ import json import mimetypes import re from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from graphlib import CycleError, TopologicalSorter from itertools import chain, groupby, islice from os import PathLike @@ -234,6 +235,199 @@ def is_non_content_values_set(message: AllMessageValues) -> bool: _IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"}) _IMAGE_SCAN_MAX_DEPTH: Final = 4 +_MEDIA_CONTENT_PART_TYPES: Final = _IMAGE_CONTENT_PART_TYPES | {"input_audio"} +_REMOVED_MEDIA_PART: Final = object() + + +@dataclass(frozen=True, slots=True) +class _ContentMediaSplit: + content: object + media: tuple[Mapping[str, object], ...] = () + text: str = "" + opaque_document_sizes: tuple[int, ...] = () + + +def _message_content_parts(value: object) -> Sequence[object] | None: + if not isinstance(value, (list, tuple)): + return None + return cast(Sequence[object], value) # cast-ok: finite JSON content arrays + + +def _file_media_part(part: Mapping[str, object]) -> Mapping[str, object] | None: + from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type + + file: Final = as_str_mapping(part.get("file")) if part.get("type") == "file" else part + if file is None: + return None + data: Final = file.get("file_data") + data_string: Final = data if isinstance(data, str) else "" + declared_type: Final = next( + (value for key in ("format", "mime_type", "content_type") if isinstance(value := file.get(key), str) and value), + "", + ) + mime_type: Final = (declared_type or parse_mime_type(data_string[:256]) or "").split(";", 1)[0].strip().lower() + if mime_type.startswith("image/"): + file_id: Final = file.get("file_id") + url: Final = data_string or (file_id if isinstance(file_id, str) else "") + image_url: Final = ( + url + if not data_string or url.startswith(("data:", "https://", "http://", "gs://")) + else f"data:{mime_type};base64,{url}" + ) + return {"type": "image_url", "image_url": {"url": image_url, "detail": file.get("detail", "auto")}} + if mime_type.startswith("audio/"): + header_end: Final = data_string.find(",", 0, 256) + encoded: Final = ( + data_string[header_end + 1 :] + if header_end >= 0 and data_string[:header_end].endswith(";base64") + else "" + if data_string.startswith(("data:", "https://", "http://", "gs://")) + else data_string + ) + return {"type": "input_audio", "input_audio": {"data": encoded, "format": mime_type.removeprefix("audio/")}} + return None + + +def _opaque_document_size(payload: object) -> int: + fields: Final = as_str_mapping(payload) + strings: Final = {key: value for key, value in (fields or {}).items() if isinstance(value, str)} + return len(str(strings)) + + +def _split_media_leaf(part: object) -> _ContentMediaSplit: + if isinstance(part, str): + return _ContentMediaSplit(part, text=part) + block: Final = as_str_mapping(part) + if block is None: + return _ContentMediaSplit(part) + part_type: Final = block.get("type") + media: Final = ( + block + if isinstance(part_type, str) and part_type in _MEDIA_CONTENT_PART_TYPES + else _file_media_part(block) + if part_type in ("file", "input_file") + else None + ) + if media is not None: + return _ContentMediaSplit(_REMOVED_MEDIA_PART, (media,)) + text: Final = block.get("text") + return _ContentMediaSplit( + part, + text=text if isinstance(text, str) else "", + opaque_document_sizes=( + (_opaque_document_size(block.get("file") if part_type == "file" else block),) + if part_type in ("file", "input_file") + else () + ), + ) + + +def _split_media_content(content: object, split_part: Callable[[object], _ContentMediaSplit]) -> _ContentMediaSplit: + parts: Final = _message_content_parts(content) + if parts is None: + return _ContentMediaSplit(content, text=content if isinstance(content, str) else "") + splits: Final = tuple(map(split_part, parts)) + media: Final = tuple(chain.from_iterable(split.media for split in splits)) + kept: Final = tuple(split.content for split in splits if split.content is not _REMOVED_MEDIA_PART) + sanitized: Final = content if not media else kept if isinstance(content, tuple) else list(kept) + return _ContentMediaSplit( + sanitized, + media, + "".join(split.text for split in splits), + tuple(chain.from_iterable(split.opaque_document_sizes for split in splits)), + ) + + +def _split_document_media(part: object) -> _ContentMediaSplit: + block: Final = as_str_mapping(part) + if block is None or block.get("type") != "document": + return _split_media_leaf(part) + source: Final = as_str_mapping(block.get("source")) + metadata: Final = "".join(value for key in ("title", "context") if isinstance(value := block.get(key), str)) + if source is None: + return _ContentMediaSplit(part, text=metadata, opaque_document_sizes=(0,)) + if source.get("type") == "text": + text: Final = source.get("data") + return _ContentMediaSplit(part, text=metadata + (text if isinstance(text, str) else "")) + if source.get("type") != "content": + return _ContentMediaSplit(part, text=metadata, opaque_document_sizes=(_opaque_document_size(source),)) + split: Final = _split_media_content(source.get("content"), _split_media_leaf) + sanitized: Final = {**block, "source": {**source, "content": split.content}} if split.media else part + return _ContentMediaSplit(sanitized, split.media, metadata + split.text, split.opaque_document_sizes) + + +def _split_tool_result_media(part: object) -> _ContentMediaSplit: + block: Final = as_str_mapping(part) + if block is None or block.get("type") != "tool_result": + return _split_document_media(part) + split: Final = _split_media_content(block.get("content"), _split_document_media) + sanitized: Final = {**block, "content": split.content} if split.media else part + return _ContentMediaSplit(sanitized, split.media, split.text, split.opaque_document_sizes) + + +def _split_message_content(message: object) -> _ContentMediaSplit: + fields: Final = as_str_mapping(message) + if fields is None: + return _ContentMediaSplit(message) + split: Final = _split_media_content(fields.get("content"), _split_tool_result_media) + sanitized: Final = {**fields, "content": split.content} if split.media else message + return _ContentMediaSplit(sanitized, split.media, split.text, split.opaque_document_sizes) + + +def split_message_media(messages: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + """Split declared media from content without mutation, following at most tool-result and document nesting.""" + parts: Final = _message_content_parts(messages) + if parts is None: + return messages, () + splits: Final = tuple(map(_split_message_content, parts)) + media: Final = tuple(chain.from_iterable(split.media for split in splits)) + if not media: + return messages, () + sanitized: Final = tuple(split.content for split in splits) + return (sanitized if isinstance(messages, tuple) else list(sanitized)), media + + +def _message_tool_argument_text(message: Mapping[str, object]) -> str: + if message.get("role") != "assistant": + return "" + functions: Final = chain( + (message.get("function_call"),), + ( + call_fields.get("function") + for call in _message_content_parts(message.get("tool_calls")) or () + if (call_fields := as_str_mapping(call)) is not None + ), + ) + return "".join( + arguments + for function in functions + if (function_fields := as_str_mapping(function)) is not None + and isinstance(arguments := function_fields.get("arguments"), str) + ) + + +def get_token_count_text_from_messages(messages: object, *, include_tool_arguments: bool = False) -> str: + """Visible message content, excluding media bytes and opaque document/reasoning payloads.""" + text, _ = get_token_count_text_and_document_sizes(messages, include_tool_arguments=include_tool_arguments) + return text + + +def get_token_count_text_and_document_sizes( + messages: object, *, include_tool_arguments: bool = False +) -> tuple[str, tuple[int, ...]]: + """Visible text and serialized opaque payload sizes, without decoding or fetching documents.""" + fields: Final = tuple( + message for value in _message_content_parts(messages) or () if (message := as_str_mapping(value)) is not None + ) + splits: Final = tuple(_split_message_content(message) for message in fields) + text: Final = "".join( + split.text + + extract_search_results_text(message.get("search_results")) + + (_message_tool_argument_text(message) if include_tool_arguments else "") + for message, split in zip(fields, splits) + ) + return text, tuple(chain.from_iterable(split.opaque_document_sizes for split in splits)) + def _content_parts_contain_image(parts: Sequence[object]) -> bool: """Depth-bounded frontier walk over nested content lists, iterative because the repo bans diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 90e79d4d853..c2eef759e07 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,15 +3,18 @@ import base64 import io import math +import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, cast import anyio import anyio.lowlevel import httpx import tiktoken +from pydantic import TypeAdapter from typing_extensions import ParamSpec, TypeVar import litellm @@ -35,6 +38,7 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.chat_template_tools import ( ChatTemplateToolSection, chat_template_tool_section, @@ -59,7 +63,6 @@ from litellm.types.llms.anthropic import ( from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionDocumentObject, - ChatCompletionNamedToolChoiceParam, ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolParam, @@ -184,22 +187,22 @@ def resize_image_high_res( # Test the function with the given example def calculate_tiles_needed( - resized_width, - resized_height, - tile_width=MAX_TILE_WIDTH, - tile_height=MAX_TILE_HEIGHT, -): + resized_width: int, + resized_height: int, + tile_width: int = MAX_TILE_WIDTH, + tile_height: int = MAX_TILE_HEIGHT, +) -> int: tiles_across: Final = (resized_width + tile_width - 1) // tile_width tiles_down: Final = (resized_height + tile_height - 1) // tile_height total_tiles: Final = tiles_across * tiles_down return total_tiles -def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int: +def high_detail_image_token_upper_bound(base_tokens: int = 85, tile_tokens: int | None = None) -> int: largest_tile_count: Final = calculate_tiles_needed( MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES ) - return base_tokens + (base_tokens * 2) * largest_tile_count + return base_tokens + (base_tokens * 2 if tile_tokens is None else tile_tokens) * largest_tile_count def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]: @@ -261,6 +264,10 @@ def get_image_dimensions( _header, encoded = data.split(",", 1) img_data = base64.b64decode(encoded) + return _get_image_dimensions_from_bytes(img_data) or (DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT) + + +def _get_image_dimensions_from_bytes(img_data: bytes) -> tuple[int, int] | None: img_type: Final = get_image_type(img_data) if img_type == "png": @@ -303,8 +310,7 @@ def get_image_dimensions( h = ((bits >> 14) & 0x3FFF) + 1 return w, h - # return sensible default image dimensions if unable to get dimensions - return DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT + return None def calculate_img_tokens( @@ -342,6 +348,161 @@ def calculate_img_tokens( return total_tokens +_IMAGE_HEADER_BASE64_CHARS: Final = 65_536 +_MAX_IMAGE_PATCHES: Final = 30_000 +_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType( + { + "gpt-4o": (85, 170), + "gpt-4.1": (85, 170), + "gpt-4o-mini": (2833, 5667), + "gpt-5": (70, 140), + "gpt-5.1": (70, 140), + "o1": (75, 150), + "o1-pro": (75, 150), + "o3": (75, 150), + } +) +_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType( + { + "gpt-6-astra": 1.2, + "gpt-5.6-sol": 1.2, + "gpt-5.6-terra": 1.2, + "gpt-5.6-luna": 1.2, + "gpt-5.5": 1.2, + "gpt-5.4": 1.2, + "gpt-5.4-mini": 1.2, + "gpt-5.4-nano": 1.2, + "gpt-5.2": 1.2, + "gpt-4.1-mini": 1.62, + } +) + + +def _inline_image_dimensions(data: str) -> tuple[int, int] | None: + separator: Final = data.find(",", 0, 256) + # Adapters can declare the image MIME separately from a generic data URL. + # The bounded byte parser still requires a supported image signature. + if not data.startswith("data:") or separator < 0 or not data[:separator].endswith(";base64"): + return None + encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS] + try: + header: Final = base64.b64decode(encoded_header, validate=True) + dimensions: Final = _get_image_dimensions_from_bytes(header) + except (ValueError, TypeError, struct.error): + return None + if dimensions is None: + return None + width, height = dimensions + return (width, height) if width > 0 and height > 0 else None + + +def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None: + if model in ("gpt-5.2", "gpt-4.1-mini"): + return None if detail == "original" else (2048, 6144) + original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5") + effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail + if effective_detail == "low": + return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None) + if effective_detail == "high": + return (65_535 if model == "gpt-6-astra" else 2048, 2500) + return ( + (65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000) + ) + + +def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int: + dimension_scale: Final = min(1.0, max_dimension / max(width, height)) + fitted_width: Final = max(1, round(width * dimension_scale)) + fitted_height: Final = max(1, round(height * dimension_scale)) + patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32) + if budget is None or patches <= budget: + return patches + shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height)) + width_patches: Final = fitted_width * shrink / 32 + height_patches: Final = fitted_height * shrink / 32 + aligned_shrink: Final = shrink * min( + math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches + ) + resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink)) + resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink)) + return math.ceil(resized_width / 32) * math.ceil(resized_height / 32) + + +def _image_policy_model_name(model: str) -> str | None: + provider, separator, provider_model = model.partition("/") + if not separator: + return model + if provider == "openai": + return provider_model + if provider == "openrouter": + return provider_model.removeprefix("openai/") if provider_model.startswith("openai/") else None + if provider not in ("azure", "azure_ai"): + return None + catalog_entry: Final = as_str_mapping( + cast( # cast-ok: read-only view of the legacy string-keyed registry; as_str_mapping checks the entry + Mapping[str, object], litellm.model_cost + ).get(model) + ) + if catalog_entry is not None and catalog_entry.get("litellm_provider") in ("azure", "azure_ai"): + return provider_model.rsplit("/", 1)[-1] + return provider_model.removeprefix("responses/").removeprefix("o_series/").removeprefix("openai/") + + +def _image_model_family(model: str) -> str | None: + provider_model: Final = _image_policy_model_name(model) + if provider_model is None: + return None + family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model) + canonical: Final = litellm.azure_llms.get(f"azure/{family}", family) if model.startswith("azure/") else family + return canonical if canonical in _TILE_IMAGE_TOKEN_COSTS or canonical in _PATCH_IMAGE_MULTIPLIERS else None + + +def _image_policy_tokens(family: str | None, dimensions: tuple[int, int] | None, detail: str) -> int: + if family is None: + return DEFAULT_IMAGE_TOKEN_COUNT + tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family) + multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family) + if tile_costs is not None: + base_tokens, tile_tokens = tile_costs + if detail == "original": + return DEFAULT_IMAGE_TOKEN_COUNT + if detail == "low": + return base_tokens + if dimensions is None: + return high_detail_image_token_upper_bound(base_tokens, tile_tokens) + width, height = dimensions + scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height)) + tiles: Final = calculate_tiles_needed(max(1, math.floor(width * scale)), max(1, math.floor(height * scale))) + return base_tokens + tiles * tile_tokens + limits: Final = _patch_image_limits(family, detail) + if limits is None or multiplier is None: + return DEFAULT_IMAGE_TOKEN_COUNT + max_dimension, patch_budget = limits + patches: Final = ( + min(math.ceil(max_dimension / 32) ** 2, patch_budget if patch_budget is not None else _MAX_IMAGE_PATCHES) + if dimensions is None + else _image_patch_count(*dimensions, max_dimension, patch_budget) + ) + return math.ceil(patches * multiplier) + + +def estimate_image_tokens( + model: str | Sequence[str], data: str, detail: str = "auto", *, fallback_model: str | None = None +) -> int: + """Estimate image policies from at most 48 KiB of inline headers, without fetching media.""" + if detail not in ("low", "high", "auto", "original"): + return DEFAULT_IMAGE_TOKEN_COUNT + models: Final = (model,) if isinstance(model, str) else model + fallback_family: Final = _image_model_family(fallback_model) if fallback_model is not None else None + families: Final = tuple( + dict.fromkeys(_image_model_family(candidate) or fallback_family for candidate in models or ("",)) + ) + if all(family is None for family in families): + return DEFAULT_IMAGE_TOKEN_COUNT + dimensions: Final = _inline_image_dimensions(data) + return max(_image_policy_tokens(family, dimensions, detail) for family in families) + + TokenCounterFunction = Callable[[str], int] """ Type for a function that counts tokens in a string. @@ -351,6 +512,7 @@ EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") +_TOOL_CHOICE_FUNCTION: Final = TypeAdapter(Mapping[str, object]) _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") ) @@ -418,10 +580,10 @@ def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, text: str | list[str] | None = None, - messages: Sequence[AllMessageValues | Message] | None = None, + messages: Sequence[object] | None = None, count_response_tokens: bool | None = False, - tools: list[ChatCompletionToolParam] | None = None, - tool_choice: ChatCompletionNamedToolChoiceParam | None = None, + tools: Sequence[object] | None = None, + tool_choice: object = None, use_default_image_token_count: bool | None = False, default_token_count: int | None = None, ) -> int: @@ -600,8 +762,8 @@ def _count_messages( def _count_extra( count_function: TokenCounterFunction, - tools: list[ChatCompletionToolParam] | None, - tool_choice: ChatCompletionNamedToolChoiceParam | None, + tools: Sequence[object] | None, + tool_choice: object, includes_system_message: bool, tool_section: ChatTemplateToolSection | None, ) -> int: @@ -638,12 +800,13 @@ def _count_tool_definitions( def _count_tool_choice( count_function: TokenCounterFunction, - tool_choice: ChatCompletionNamedToolChoiceParam | None, + tool_choice: object, ) -> int: if tool_choice == "none": return 1 if isinstance(tool_choice, dict): - return 7 + count_function(str(tool_choice["function"]["name"])) + function: Final = _TOOL_CHOICE_FUNCTION.validate_python(tool_choice["function"]) + return 7 + count_function(str(function["name"])) return 0 diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 567b8452ae2..74b858ebfdd 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -42,14 +42,22 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DEFAULT_IMAGE_TOKEN_COUNT, + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages, + as_openai_image_part, + get_token_count_text_and_document_sizes, + get_token_count_text_from_messages, + split_message_media, ) from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count from litellm.llms.azure.ptu_capacity import normalized_tokens from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( @@ -73,7 +81,6 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit -from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, @@ -98,6 +105,7 @@ if TYPE_CHECKING: from opentelemetry.trace import Span as _Span from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache + from litellm.router import Router from litellm.types.agents import AgentResponse from litellm.types.caching import RedisPipelineIncrementOperation @@ -109,6 +117,9 @@ else: _REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object]) +_TOKEN_COUNT_ITEMS: Final = TypeAdapter(list[object]) +_TOKEN_COUNT_SEQUENCE: Final = TypeAdapter[Sequence[object] | None](Sequence[object] | None) +_TOKEN_COUNT_TEXT: Final = TypeAdapter[str | list[str] | None](str | list[str] | None) _MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None) @@ -135,6 +146,24 @@ class RateLimitedModel: return requested_limit if requested_limit is not None else limits.get(self.group) +@dataclass(frozen=True, slots=True) +class RequestInputTokenEstimate: + cheap_tokens: int + precise_tokens: int | None = None + + +def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]: + if router is None: + return (model,) + return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,) + + +def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]: + from litellm.proxy.proxy_server import llm_router + + return _resolve_media_models(llm_router, model, team_id) + + def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: from litellm.proxy.proxy_server import llm_router @@ -502,18 +531,6 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( CallTypes.agenerate_content_stream.value, ) RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16 -# litellm.token_counter has no per-type handling for "input_audio" content -# blocks (unlike images, which use use_default_image_token_count) -- it -# silently contributes 0 tokens for them. When the block carries a base64 -# payload, the estimate is derived from the decoded byte count; when the -# block is a reference without a payload (or the payload is missing), this -# flat per-block floor is used instead. -DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300 -# Conservative bytes-per-token assumption for size-based audio estimation: -# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing -# the lowest reasonable bitrate means we never under-reserve for higher- -# quality audio recorded at the same wall-clock duration. -_AUDIO_BYTES_PER_TOKEN: Final = 1600 # Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from # "model_per_project" (the combined-TPM descriptor) so both can be enforced # on the same project+model simultaneously without colliding on cache keys. @@ -867,11 +884,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ] = _resolve_ptu_team_ceiling_via_proxy_router, fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings, force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings, + media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now self._tag_rate_limit_resolver = tag_rate_limit_resolver self._model_group_resolver = model_group_resolver + self._media_model_resolver = media_model_resolver self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver @@ -1138,6 +1157,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit: int | None = None, call_type: str | None = None, configured_output_tokens: int | None = None, + input_estimate: RequestInputTokenEstimate | None = None, ) -> int: """ Estimate total tokens this request will consume so we can reserve them @@ -1159,11 +1179,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): floor entirely, so the reservation reflects what this tenant's model actually emits rather than one constant shared by every tenant. """ + measured_input: Final = ( + input_estimate + if input_estimate is not None + else self._estimate_request_input_tokens(_REQUEST_RATE_LIMIT_DATA.validate_python(data), model, call_type) + ) estimated_input_tokens, max_tokens_estimate = self._estimate_input_and_output_tokens( data=data, min_configured_tpm_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + input_estimate=measured_input, ) total_estimated: Final = estimated_input_tokens + max_tokens_estimate @@ -1184,6 +1210,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type: str | None, configured_output_tokens: int | None, raw_estimate: int, + input_estimate: RequestInputTokenEstimate | None = None, ) -> int: """The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the output budget the way the ceiling does instead of the raw sum the other scopes reserve.""" @@ -1194,6 +1221,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + input_estimate=input_estimate, ) normalized: Final = normalized_tokens( ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate @@ -1206,6 +1234,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit: int | None = None, call_type: str | None = None, configured_output_tokens: int | None = None, + input_estimate: RequestInputTokenEstimate | None = None, ) -> tuple[int, int]: """ Estimate input tokens and output (max_tokens) budget separately, so @@ -1231,43 +1260,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ if not isinstance(data, dict): return 0, 0 - translated_data: Final = self._translate_google_genai_native_request(data, call_type) - estimable_data: Final = translated_data if translated_data is not None else data - selected_fields: Final[tuple[object | None, object | None, object | None]] = ( - (None, None, estimable_data.get("input")) - if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES - else (None, estimable_data.get("prompt"), None) - if call_type in TEXT_COMPLETION_API_CALL_TYPES - else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(estimable_data)), None) - if call_type in EXTRACTION_API_CALL_TYPES - else (estimable_data.get("messages"), None, None) - if call_type - else ( - estimable_data.get("messages"), - estimable_data.get("prompt"), - estimable_data.get("input"), + estimated_input_tokens: Final = ( + input_estimate + if input_estimate is not None + else self._estimate_request_input_tokens( + _REQUEST_RATE_LIMIT_DATA.validate_python(data), call_type=call_type ) - ) - messages, prompt, raw_input = selected_fields - input_text: Final = ( - billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input - ) - - total_chars: Final = ( - len(get_str_from_messages(messages)) - if isinstance(messages, list) and messages - else len(prompt) - if isinstance(prompt, str) - else sum(len(str(item)) for item in prompt) - if isinstance(prompt, list) - else len(input_text) - if isinstance(input_text, str) - else sum(len(str(item)) for item in input_text) - if isinstance(input_text, list) - else 0 - ) - - estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + ).cheap_tokens explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type) generates_no_output: Final = ( @@ -1283,7 +1282,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): max_tokens_estimate: Final = ( 0 if generates_no_output - or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None) + or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None) else explicit_max_tokens if explicit_max_tokens is not None else configured_output_tokens @@ -3753,72 +3752,37 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) @staticmethod - def _estimate_audio_block_tokens(block: object) -> int: - """ - Token estimate for one ``input_audio`` content block. - - When the block carries a base64 ``data`` payload, the estimate comes - from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``), - assuming the lowest reasonable audio bitrate so we never under-reserve - for higher-quality recordings of the same duration. - - When no payload is present (reference-only block or missing ``data``), - falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``. - """ - if not isinstance(block, dict): - return DEFAULT_AUDIO_TOKEN_ESTIMATE - input_audio: Final = block.get("input_audio") - b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None - if b64_data and isinstance(b64_data, str): - decoded_bytes: Final = len(b64_data) * 3 // 4 - return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE) - return DEFAULT_AUDIO_TOKEN_ESTIMATE - - @classmethod - def _estimate_audio_content_tokens(cls, messages: object) -> int: - """ - Sum of per-block audio token estimates across all ``messages``. - Returns 0 when there are no ``input_audio`` blocks, which the caller - uses to skip the (relatively expensive) strip pass. - """ - if not isinstance(messages, list): - return 0 - return sum( - cls._estimate_audio_block_tokens(block) - for message in messages - if isinstance(message, dict) - for content in (message.get("content"),) - if isinstance(content, list) - for block in content - if isinstance(block, dict) and block.get("type") == "input_audio" + def _estimate_image_block_tokens( + block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None + ) -> int: + normalized: Final = as_openai_image_part(block) + image: Final = _callback_kwargs_mapping(normalized.get("image_url") if normalized is not None else None) + url: Final = image.get("url") + detail: Final = _callback_kwargs_mapping(block.get("image_url")).get("detail") or block.get("detail") + return estimate_image_tokens( + models, + url if isinstance(url, str) else "", + detail if isinstance(detail, str) else "auto", + fallback_model=fallback_model, ) @staticmethod - def _strip_audio_content_blocks(messages: object) -> object: - """ - Drop ``input_audio`` content blocks before passing ``messages`` to - ``token_counter``, which raises ``ValueError`` on them (no per-type - handling, unlike images). The audio contribution is added back - separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the - message (text/images/tools) still gets counted accurately instead of - the whole call falling back to the cheap char-count estimate. - """ - if not isinstance(messages, list): - return messages - sanitized: Final[list[object]] = [] # mutable-ok: token_counter requires a list of message dicts - for message in messages: - if not isinstance(message, dict): - sanitized.append(message) - continue - content = message.get("content") - if not isinstance(content, list): - sanitized.append(message) - continue - filtered_content = [ - block for block in content if not (isinstance(block, dict) and block.get("type") == "input_audio") - ] - sanitized.append({**message, "content": filtered_content}) - return sanitized + def _estimate_audio_block_tokens( + block: object, models: tuple[str, ...] = (), fallback_model: str | None = None + ) -> int: + audio: Final = _callback_kwargs_mapping(_callback_kwargs_mapping(block).get("input_audio")) + data: Final = audio.get("data") + return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model) + + @staticmethod + def _requires_input_count_offload(data: Mapping[str, object], call_type: str | None) -> bool: + if call_type in RESPONSES_API_CALL_TYPES + GOOGLE_GENAI_NATIVE_CALL_TYPES: + return True + messages: Final = data.get("messages") + return isinstance(messages, list) and any( + isinstance(_callback_kwargs_mapping(message).get("content"), list) + for message in _TOKEN_COUNT_ITEMS.validate_python(messages) + ) @staticmethod def _responses_input_to_chat_messages(data: object) -> Sequence[object]: @@ -3873,93 +3837,110 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): state: Final = data.get("state") return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}" - def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: - """ - Model-aware input token estimate for the project ITPM reservation, - using ``litellm.token_counter`` -- the same approach the - deployment-level itpm/otpm check uses in - ``io_token_rate_limit_check.py``. Unlike the cheap char-count - estimate the combined-TPM path uses, this accounts for image/tool - content and derives per-``input_audio``-block estimates from the - base64 payload size (assuming the lowest reasonable bitrate so - longer recordings always reserve proportionally more), so a burst - of multimodal, tool-heavy, or audio-heavy requests can't each - reserve only the one-token floor and blow past ITPM before - post-call reconciliation catches up. - - For the Responses API, ``input`` is converted to chat messages first - (via ``_responses_input_to_chat_messages``) so its own multimodal - content blocks are counted the same way; ``token_counter``'s ``text`` - argument can only see plain strings in a list, not content blocks. - - Falls back to the cheap char-count estimate if ``token_counter`` - can't resolve a tokenizer for this model (e.g. an unrecognized - custom model name) or otherwise raises -- the audio add-on still - applies on top of the fallback. - """ + def _estimate_request_input_tokens( + self, + data: object, + model: str | None = None, + call_type: str | None = None, + models: tuple[str, ...] = (), + precise: bool = False, + ) -> RequestInputTokenEstimate: from litellm import token_counter - if not isinstance(data, dict): - return 0 + if not isinstance(data, Mapping): + return RequestInputTokenEstimate(0, 0 if precise else None) + request: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(data) is_responses_request: Final = call_type in RESPONSES_API_CALL_TYPES - translated_request: Final = ( - None if is_responses_request else self._translate_google_genai_native_request(data, call_type) + translated: Final = ( + None if is_responses_request else self._translate_google_genai_native_request(request, call_type) ) - is_embedding_request: Final = self._is_embedding_request(data, call_type) - embedding_text: Final = data.get("input") if is_embedding_request else None - pretokenized_input_tokens: Final = ( - self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None - ) - if pretokenized_input_tokens is not None: - return pretokenized_input_tokens + embedding_input: Final = request.get("input") if self._is_embedding_request(request, call_type) else None + pretokenized_tokens: Final = self._count_pretokenized_embedding_input(embedding_input) + is_rerank_request: Final = call_type in RERANK_API_CALL_TYPES - prompt: Final = data.get("prompt") - fallback_text: Final = prompt if prompt is not None else data.get("input") - selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = ( - (self._responses_input_to_chat_messages(data), None, data.get("tools"), data.get("tool_choice")) + prompt: Final = request.get("prompt") + selected: Final[tuple[object, object, object, object]] = ( + (self._responses_input_to_chat_messages(request), None, request.get("tools"), request.get("tool_choice")) if is_responses_request - else ( - translated_request.get("messages"), - None, - translated_request.get("tools"), - translated_request.get("tool_choice"), - ) - if translated_request is not None - else (None, embedding_text, data.get("tools"), data.get("tool_choice")) - if is_embedding_request - else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice")) - if call_type in RERANK_API_CALL_TYPES - else (None, prompt, data.get("tools"), data.get("tool_choice")) + else (translated.get("messages"), None, translated.get("tools"), translated.get("tool_choice")) + if translated is not None + else (None, embedding_input, request.get("tools"), request.get("tool_choice")) + if self._is_embedding_request(request, call_type) + else (None, self._rerank_input_to_text(request), request.get("tools"), request.get("tool_choice")) + if is_rerank_request + else (None, prompt, request.get("tools"), request.get("tool_choice")) if call_type in TEXT_COMPLETION_API_CALL_TYPES - else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(data)), None, None) + else (None, self._extraction_input_text(request), None, None) if call_type in EXTRACTION_API_CALL_TYPES - else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice")) - ) - messages, selected_text, countable_tools, countable_tool_choice = selected_inputs - - audio_token_estimate: Final = self._estimate_audio_content_tokens(messages) - countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages - - try: - estimate: Final = max( - 0, - int( - token_counter( - model=model or "", - messages=countable_messages, - text=selected_text, - tools=countable_tools, - tool_choice=countable_tool_choice, - use_default_image_token_count=True, - ) - ), + else ( + request.get("messages"), + None if call_type else prompt if prompt is not None else request.get("input"), + request.get("tools"), + request.get("tool_choice"), ) - return estimate + audio_token_estimate - except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate - if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str): - return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN) - estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type) - return estimated_input_tokens + audio_token_estimate + ) + messages, text, tools, tool_choice = selected + countable_messages, media = split_message_media(messages) + model_value: Final = request.get("model") + fallback_model: Final = model if model is not None else model_value if isinstance(model_value, str) else None + media_tokens: Final = sum( + self._estimate_audio_block_tokens(block, models, fallback_model) + if block.get("type") == "input_audio" + else self._estimate_image_block_tokens(block, models, fallback_model) + for block in media + ) + message_text, opaque_document_sizes = get_token_count_text_and_document_sizes( + countable_messages, include_tool_arguments=is_responses_request + ) + text_chars: Final = ( + len(message_text) + if isinstance(messages, list) and messages + else len(text) + if isinstance(text, str) + else sum(len(str(item)) for item in _TOKEN_COUNT_ITEMS.validate_python(text)) + if isinstance(text, list) + else 0 + ) + cheap_chars: Final = ( + len(get_token_count_text_from_messages(request.get("messages"))) if is_rerank_request else text_chars + ) + # Keep opaque bytes' character fallback until precise counting can read their pages. + cheap_tokens: Final = ( + (max(1, cheap_chars // DEFAULT_CHARS_PER_TOKEN) if cheap_chars > 0 else 0) + + media_tokens + + sum( + max(DEFAULT_IMAGE_TOKEN_COUNT, (size + DEFAULT_CHARS_PER_TOKEN - 1) // DEFAULT_CHARS_PER_TOKEN) + for size in opaque_document_sizes + ) + ) + if not precise: + return RequestInputTokenEstimate(cheap_tokens) + if pretokenized_tokens is not None: + return RequestInputTokenEstimate(cheap_tokens, pretokenized_tokens) + try: + precise_tokens: Final = ( + max( + 0, + int( + token_counter( + model=fallback_model or "", + messages=_TOKEN_COUNT_SEQUENCE.validate_python(countable_messages), + text=_TOKEN_COUNT_TEXT.validate_python(text), + tools=_TOKEN_COUNT_SEQUENCE.validate_python(tools), + tool_choice=tool_choice, + use_default_image_token_count=True, + ) + ), + ) + + media_tokens + ) + except Exception: # noqa: BLE001 # tokenizer failures use the already measured input + fallback_tokens: Final = text_chars // DEFAULT_CHARS_PER_TOKEN if is_rerank_request else cheap_tokens + return RequestInputTokenEstimate(cheap_tokens, fallback_tokens) + return RequestInputTokenEstimate(cheap_tokens, precise_tokens) + + def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: + return self._estimate_request_input_tokens(data, model, call_type, precise=True).precise_tokens or 0 async def _reserve_project_io_tokens_or_raise( self, @@ -3972,6 +3953,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type: str | None = None, endpoint_type: EndpointType = EndpointType.GENERIC, reuses_request_slot: bool = False, + input_estimate: RequestInputTokenEstimate | None = None, ) -> None: """ Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style @@ -3999,15 +3981,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if v is not None ] min_configured_otpm_limit: Final = min(configured_otpm_limits) if configured_otpm_limits else None + measured_input: Final = ( + input_estimate + if input_estimate is not None + else await offload_token_count(self._estimate_request_input_tokens)( + data=_REQUEST_RATE_LIMIT_DATA.validate_python(data), + model=requested_model, + call_type=call_type, + models=self._media_model_resolver(requested_model or "", user_api_key_dict.team_id), + precise=any(d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY for d in io_token_descriptors), + ) + ) _, raw_estimated_output_tokens = self._estimate_input_and_output_tokens( data=data, min_configured_tpm_limit=min_configured_otpm_limit, call_type=call_type, + input_estimate=measured_input, ) - raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=requested_model, call_type=call_type + estimated_input_tokens: Final = max( + measured_input.precise_tokens if measured_input.precise_tokens is not None else measured_input.cheap_tokens, + 1, ) - estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( raw_estimated_output_tokens if self._has_explicit_output_cap(data, call_type) @@ -4323,6 +4317,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) has_tpm_limits: Final = bool(configured_tpm_limits) + needs_precise_input: Final = any(d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY for d in descriptors) + needs_input_estimate: Final = (has_tpm_limits and self.tpm_reservation_enabled) or any( + d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for d in descriptors + ) + offload_input: Final = self._requires_input_count_offload(request_data, call_type) + input_estimate: Final = ( + await offload_token_count(self._estimate_request_input_tokens)( + data=request_data, + model=requested_model, + call_type=call_type, + models=self._media_model_resolver(requested_model or "", user_api_key_dict.team_id) + if offload_input + else (), + precise=needs_precise_input, + ) + if needs_input_estimate and (offload_input or needs_precise_input) + else self._estimate_request_input_tokens(request_data, requested_model, call_type) + if needs_input_estimate + else None + ) + # Populated on a successful combined-TPM reservation below, so the # project ITPM/OTPM block further down can roll it back if a # different bucket in the same request subsequently hits its @@ -4366,6 +4381,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + input_estimate=input_estimate, ), 1, ) @@ -4390,6 +4406,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type=call_type, configured_output_tokens=configured_output_tokens, raw_estimate=estimated_tokens, + input_estimate=input_estimate, ) if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit: if not reuses_request_slot: @@ -4453,6 +4470,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): user_api_key_dict=user_api_key_dict, tpm_reservation_scopes=tpm_reservation_scopes, tpm_reservation_amount=tpm_reservation_amount, + input_estimate=input_estimate, call_type=call_type, endpoint_type=endpoint_type, reuses_request_slot=reuses_request_slot, diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..1b656d8616b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10224,28 +10224,38 @@ class Router: model_info[field] = backend_value @staticmethod - def _cost_map_backend_model(deployment: Deployment) -> str: - model_info_base_model: Final = deployment.model_info.base_model + def _cost_map_backend_model(deployment: Deployment | DeploymentTypedDict) -> str: + model_info_base_model: Final = ( + deployment.model_info.base_model + if isinstance(deployment, Deployment) + else _configured_model_info(deployment).get("base_model") + ) if isinstance(model_info_base_model, str) and model_info_base_model: return model_info_base_model - params_base_model: Final = deployment.litellm_params.get("base_model") + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python( + deployment.litellm_params.model_dump(exclude_none=True) + if isinstance(deployment, Deployment) + else deployment["litellm_params"] + ) + params_base_model: Final = params.get("base_model") if isinstance(params_base_model, str) and params_base_model: return params_base_model - params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + model: Final = ( + deployment.litellm_params.model + if isinstance(deployment, Deployment) + else deployment["litellm_params"].get("model", "") + ) invoked_model: Final = params.get("model_id") if ( isinstance(invoked_model, str) and invoked_model - and ( - deployment.litellm_params.custom_llm_provider == "bedrock" - or deployment.litellm_params.model.startswith("bedrock/") - ) + and (params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/")) ): profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}" profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock") if not isinstance(profile_info, ModelInfoLookupMiss): return profile_model - return deployment.litellm_params.model + return model @staticmethod def _inherit_builtin_service_tier_pricing( @@ -13795,7 +13805,9 @@ class Router: ] return named or self._get_wildcard_deployments(model_name=model_group) - def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]: + def resolved_litellm_models( + self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False + ) -> tuple[str, ...]: """The provider model strings `model_name` can actually be served by on this proxy. `get_model_list` composes every channel the request path itself uses (exact name, @@ -13808,6 +13820,12 @@ class Router: deployment behind it - so the fallback for an empty result is the caller's policy, never this function's. """ + if use_base_model: + return tuple( + backend_model + for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () + if (backend_model := self._cost_map_backend_model(deployment)) + ) return tuple( litellm_model for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () diff --git a/litellm/utils.py b/litellm/utils.py index 4315ae426cf..6a874577347 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -224,7 +224,6 @@ from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, ChatCompletionAssistantToolCall, - ChatCompletionNamedToolChoiceParam, ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk, OpenAITextCompletionUserMessage, @@ -2822,10 +2821,10 @@ def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, text: str | list[str] | None = None, - messages: Sequence | None = None, + messages: Sequence[object] | None = None, count_response_tokens: bool | None = False, - tools: list[ChatCompletionToolParam] | None = None, - tool_choice: ChatCompletionNamedToolChoiceParam | None = None, + tools: Sequence[object] | None = None, + tool_choice: object = None, use_default_image_token_count: bool | None = False, default_token_count: int | None = None, ) -> int: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 5f7c9b2842a..9cbc2c816d5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -22,6 +22,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_format_from_file_id, get_semantic_cache_prompt_from_messages, get_str_from_messages, + get_token_count_text_and_document_sizes, + get_token_count_text_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, @@ -35,6 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( schema_has_unresolvable_ref, schema_property_count, split_concatenated_json_objects, + split_message_media, strip_encrypted_reasoning_from_messages, strip_litellm_response_only_fields, system_messages_first, @@ -2465,6 +2468,169 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch assert schema_grammar_branch_count(schema) == branches +@pytest.mark.parametrize( + "media", + [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, + {"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"}, + {"type": "image", "source": {"type": "file", "file_id": "opaque-image"}}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + {"type": "image"}, + {"type": "input_audio", "input_audio": None}, + ], +) +def test_split_message_media_preserves_every_nonmedia_sibling_and_media_detail( + media: Mapping[str, object], +) -> None: + siblings: Final = ["visible", None, 7, {"type": []}, {"type": "text", "text": "caption"}] + messages: Final = [{"role": "tool", "tool_call_id": "call_1", "content": [*siblings, media, media]}] + original: Final = copy.deepcopy(messages) + + sanitized, leaves = split_message_media(messages) + + assert sanitized == [{"role": "tool", "tool_call_id": "call_1", "content": siblings}] + assert leaves == (media, media) + assert messages == original + assert split_message_media(sanitized) == (sanitized, ()) + + +def test_split_message_media_keeps_nested_tool_and_document_text_and_metadata() -> None: + direct: Final = {"type": "image_url", "image_url": "https://example.com/direct.png"} + image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}} + audio: Final = {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}} + text: Final = {"type": "text", "text": "document text"} + file: Final = {"type": "file", "file": {"file_id": "opaque-file"}} + opaque: Final = {"type": "document", "source": {"type": "url", "url": "https://example.com/document.pdf"}} + document: Final = { + "type": "document", "title": "title", "context": "context", "citations": {"enabled": True}, + "source": {"type": "content", "content": [text, image, file, "tail"]}, + } + tool_result: Final = { + "type": "tool_result", "tool_use_id": "toolu_1", "is_error": False, + "content": ["tool text", audio, document, opaque], + } + messages: Final = [ + {"role": "user", "content": [direct, tool_result]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_2", "content": "result"}, + {"type": "document", "source": {"type": "content", "content": "document"}}, + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "declared"}}, + ]}, + ] + original: Final = copy.deepcopy(messages) + + sanitized, leaves = split_message_media(messages) + + assert sanitized == [{ + "role": "user", "content": [{ + **tool_result, + "content": ["tool text", {**document, "source": {"type": "content", "content": [text, file, "tail"]}}, opaque], + }], + }, messages[1]] + assert leaves == (direct, audio, image) + expected_text: Final = "tool texttitlecontextdocument texttailresultdocumentdeclared" + assert get_token_count_text_from_messages(sanitized) == expected_text + visible_text, opaque_sizes = get_token_count_text_and_document_sizes(sanitized) + assert visible_text == expected_text + assert len(opaque_sizes) == 2 and all(size > 0 for size in opaque_sizes) + assert messages == original + + +def test_split_message_media_does_not_walk_arbitrary_tool_arguments_or_invalid_nesting() -> None: + image: Final = {"type": "image", "source": {"type": "url", "url": "https://example.com/image.png"}} + document: Final = {"type": "document", "source": {"type": "content", "content": [image]}} + tool: Final = {"type": "tool_result", "content": [image]} + blocks: Final = [ + {"type": "tool_use", "name": "inspect", "input": {"content": [image]}}, + {"type": "unknown", "content": [image]}, + {"type": "document", "source": {"type": "content", "content": [document]}}, + {"type": "tool_result", "content": [tool]}, + ] + messages: Final = [{"role": "assistant", "content": blocks}] + sanitized, leaves = split_message_media(messages) + assert sanitized == messages + assert leaves == () + + +def test_token_count_text_includes_only_opaque_assistant_argument_strings_when_requested() -> None: + arguments: Final = '{"type":"input_image","image_url":"data:image/png;base64,IMAGE"}' + calls: Final = [ + {"function": {"arguments": arguments}}, + {"function": {"arguments": {"text": "do not traverse"}}}, + {"function": None}, + None, + ] + messages: Final = [ + {"role": "assistant", "content": "before", "tool_calls": calls, "function_call": {"arguments": "legacy"}}, + {"role": "user", "content": "after", "tool_calls": calls}, + ] + original: Final = copy.deepcopy(messages) + + assert get_token_count_text_from_messages(messages) == "beforeafter" + assert get_token_count_text_from_messages(messages, include_tool_arguments=True) == ( + "beforelegacy" + arguments + "after" + ) + assert messages == original + + +@pytest.mark.parametrize( + ("part", "expected"), + [ + ( + {"type": "file", "file": {"file_data": "data:image/png;base64,IMAGE", "detail": "low"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "low"}}, + ), + ( + {"type": "input_file", "file_data": "data:image/png;base64,IMAGE", "detail": "low"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "low"}}, + ), + ( + {"type": "file", "file": {"file_id": "opaque", "mime_type": "image/jpeg"}}, + {"type": "image_url", "image_url": {"url": "opaque", "detail": "auto"}}, + ), + ( + {"type": "file", "file": {"file_data": "data:audio/wav;base64,AUDIO"}}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + ), + ( + {"type": "file", "file": {"file_data": "AUDIO", "content_type": "audio/mpeg"}}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "mpeg"}}, + ), + ( + {"type": "file", "file": {"file_id": "https://example.com/audio.wav", "format": "audio/wav"}}, + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + ), + ], +) +def test_split_message_media_normalizes_only_explicit_image_and_audio_files( + part: Mapping[str, object], expected: Mapping[str, object], +) -> None: + messages: Final = [{"role": "user", "content": [part]}] + assert split_message_media(messages) == ([{"role": "user", "content": []}], (expected,)) + assert get_token_count_text_and_document_sizes(messages) == ("", ()) + + +@pytest.mark.parametrize( + "part", + [ + {"type": "file", "file": {"file_data": "data:application/pdf;base64,PDF"}}, + {"type": "file", "file": {"file_data": "data:video/mp4;base64,VIDEO"}}, + {"type": "file", "file": {"file_id": "https://example.com/name.png"}}, + {"type": "file", "file": {"file_data": "IMAGE", "format": "application/pdf"}}, + {"type": "file", "file": None}, + {"type": "thinking", "thinking": "thought", "signature": "opaque"}, + {"type": "tool_use", "name": "read", "input": {"path": "file.txt"}}, + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}}, + ], +) +def test_split_message_media_preserves_opaque_documents_and_unclassified_files(part: Mapping[str, object]) -> None: + messages: Final = [{"role": "user", "content": [part]}] + sanitized, leaves = split_message_media(messages) + assert sanitized == messages + assert leaves == () + assert get_token_count_text_from_messages(messages) == "" + + @pytest.mark.parametrize("part_type", ["image_url", "input_image"]) @pytest.mark.parametrize( "fields", diff --git a/tests/unit/litellm_core_utils/test_audio_utils.py b/tests/unit/litellm_core_utils/test_audio_utils.py index 155f6680416..83160756f4a 100644 --- a/tests/unit/litellm_core_utils/test_audio_utils.py +++ b/tests/unit/litellm_core_utils/test_audio_utils.py @@ -2,16 +2,22 @@ Test the audio utils functionality in litellm_core_utils/audio_utils/utils.py """ +import base64 import io import os import tempfile +from typing import Final from unittest.mock import mock_open, patch import pytest +import soundfile as sf from litellm.litellm_core_utils.audio_utils.utils import ( + AUDIO_BYTES_PER_TOKEN, + DEFAULT_AUDIO_TOKEN_ESTIMATE, ProcessedAudioFile, calculate_request_duration, + estimate_audio_tokens, get_audio_file_content_hash, get_audio_file_for_health_check, get_audio_file_name, @@ -19,6 +25,74 @@ from litellm.litellm_core_utils.audio_utils.utils import ( ) +def _encoded_audio(seconds: int, sample_rate: int, channels: int, container: str) -> str: + buffer: Final = io.BytesIO() + with sf.SoundFile(buffer, "w", samplerate=sample_rate, channels=channels, format=container) as audio: + audio.buffer_write(b"\0" * (seconds * sample_rate * channels * 2), dtype="int16") + return base64.b64encode(buffer.getvalue()).decode() + + +@pytest.mark.parametrize( + ("sample_rate", "channels", "container"), + [ + (8000, 1, "WAV"), + (48000, 2, "WAV"), + (24000, 1, "FLAC"), + (48000, 2, "OGG"), + (24000, 1, "MP3"), + ], +) +def test_audio_estimate_uses_duration_across_containers_sample_rates_and_channels( + sample_rate: int, channels: int, container: str +) -> None: + short: Final = _encoded_audio(1, sample_rate, channels, container) + long: Final = _encoded_audio(31, sample_rate, channels, container) + assert estimate_audio_tokens(short) == 10 + assert estimate_audio_tokens(long) == 310 + + +# Audio rates verified 2026-10-26: https://ai.google.dev/gemini-api/docs/audio and +# https://developers.openai.com/api/docs/guides/voice-latency-cost?voice-api=realtime +@pytest.mark.parametrize( + ("models", "fallback_model", "expected"), + [ + (("gpt-audio-1.5",), None, 10), + (("gemini/gemini-3.8-flash",), None, 32), + (("vertex_ai/gemini-3.8-flash",), None, 32), + (("openrouter/google/gemini-3.8-flash",), None, 32), + (("gpt-audio-1.5", "gemini-3.8-flash"), None, 32), + (("azure/audio-deployment",), "gpt-audio-1.5", 10), + (("opaque-deployment",), "gemini-3.8-flash", 32), + (("openai/gpt-audio-1.5",), "gemini-3.8-flash", 10), + ((), "gemini-3.8-flash", 32), + (("other/gemini-3.8-flash",), None, 10), + ], +) +def test_audio_estimate_selects_the_backend_rate_before_the_requested_model( + models: tuple[str, ...], fallback_model: str | None, expected: int +) -> None: + assert estimate_audio_tokens(_encoded_audio(1, 8000, 1, "WAV"), models, fallback_model) == expected + + +@pytest.mark.parametrize("data", [None, "", "invalid-base64!", base64.b64encode(b"unreadable audio").decode()]) +def test_audio_estimate_preserves_a_nonzero_fallback_for_unreadable_audio(data: str | None) -> None: + assert estimate_audio_tokens(data) == DEFAULT_AUDIO_TOKEN_ESTIMATE + + +def test_audio_estimate_preserves_size_based_fallback_for_large_unreadable_audio() -> None: + data: Final = base64.b64encode(b"\0" * (AUDIO_BYTES_PER_TOKEN * 400)).decode() + assert estimate_audio_tokens(data) == 400 + + +@pytest.mark.parametrize("extra_bytes", [0, 1]) +def test_audio_estimate_bounds_base64_allocation_before_decoding(extra_bytes: int) -> None: + data: Final = base64.b64encode(b"\0" * (16 * 1024 * 1024 + extra_bytes)).decode() + with patch("base64.b64decode", wraps=base64.b64decode) as decode_audio: + estimate: Final = estimate_audio_tokens(data) + assert estimate == len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN + assert decode_audio.call_count == (1 if extra_bytes == 0 else 0) + + class TestProcessAudioFile: """Test the process_audio_file function with various input types""" diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 88af656ad2e..146d19eaae5 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -35,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import ( _get_extrapolating_count_function, _get_tiktoken_count_function, calculate_img_tokens, + estimate_image_tokens, high_detail_image_token_upper_bound, offload_token_count, ) @@ -2025,6 +2026,173 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_ assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() +# Image policies verified 2026-10-24: https://developers.openai.com/api/docs/guides/images-vision +@pytest.mark.parametrize( + ("model", "width", "height", "detail", "expected"), + [ + ("gpt-4o", 16, 16, "high", 255), + ("gpt-4.1", 1024, 1024, "high", 765), + ("gpt-4o-mini", 1024, 1024, "high", 25501), + ("gpt-5.1", 1024, 1024, "auto", 630), + ("gpt-5", 1024, 1024, "low", 70), + ("o1", 1024, 1024, "high", 675), + ("o1-pro", 1024, 1024, "high", 675), + ("o3", 1024, 1024, "low", 75), + ("openai/gpt-4o-2024-08-06", 2048, 4096, "auto", 1105), + ("azure/gpt-4.1", 1024, 1024, "low", 85), + ("openrouter/openai/gpt-4o-mini", 16, 16, "low", 2833), + ("gpt-6-astra", 1024, 1024, "high", 1229), + ("gpt-6-astra", 2048, 2048, "high", 3000), + ("gpt-6-astra", 4096, 512, "high", 2458), + ("gpt-6-astra", 2048, 2048, "auto", 4916), + ("gpt-6-astra", 2048, 2048, "low", 308), + ("gpt-5.6-sol", 4096, 512, "high", 615), + ("gpt-5.6-terra", 2048, 2048, "original", 4916), + ("gpt-5.6-luna", 2048, 2048, "auto", 4916), + ("gpt-5.5", 6400, 6400, "original", 12000), + ("gpt-5.4", 2048, 2048, "low", 4916), + ("gpt-5.4-mini", 2048, 2048, "auto", 3000), + ("gpt-5.4-nano", 6400, 6400, "original", 12000), + ("gpt-5.2", 4096, 512, "auto", 615), + ("gpt-4.1-mini-2025-04-14", 1024, 1024, "low", 1659), + ("gpt-6-astra", 16, 16, "original", 2), + ], +) +def test_image_estimate_uses_model_resolution_and_detail( + model: str, width: int, height: int, detail: str, expected: int +) -> None: + assert estimate_image_tokens(model, _png_data_url(width, height), detail) == expected + + +@pytest.mark.parametrize( + ("model", "data", "detail"), + [ + ("unregistered-model", _png_data_url(1024, 1024), "high"), + ("gpt-4.1-unrecognized", _png_data_url(1024, 1024), "high"), + ("other/gpt-4o", _png_data_url(1024, 1024), "high"), + ("gpt-4.1-nano", _png_data_url(1024, 1024), "high"), + ("gpt-4o", _png_data_url(1024, 1024), "original"), + ("gpt-5.2", _png_data_url(1024, 1024), "original"), + ("gpt-6-astra", _png_data_url(1024, 1024), "invalid"), + ], +) +def test_image_estimate_falls_back_when_policy_is_unavailable(model: str, data: str, detail: str) -> None: + assert estimate_image_tokens(model, data, detail) == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT + + +# Unavailable dimensions use documented limits, verified 2026-10-26: +# https://developers.openai.com/api/docs/guides/images-vision (30,000 accepted patches before the multiplier) +@pytest.mark.parametrize( + ("model", "detail", "expected", "data"), + [ + ("gpt-4o", "high", high_detail_image_token_upper_bound(), "https://image.invalid/photo.png"), + ("gpt-4o-mini", "auto", high_detail_image_token_upper_bound(2833, 5667), "https://image.invalid/photo.png"), + ("gpt-6-astra", "original", 36_000, "https://image.invalid/photo.png"), + ("gpt-6-astra", "high", 3000, "https://image.invalid/photo.png"), + ("gpt-6-astra", "low", 308, "https://image.invalid/photo.png"), + ("gpt-4o-mini", "low", 2833, "https://image.invalid/photo.png"), + ("gpt-6-astra", "high", 3000, "data:image/png;base64,invalid!"), + ("gpt-6-astra", "high", 3000, "data:application/octet-stream;base64,bm90LWFuLWltYWdl"), + ("gpt-6-astra", "high", 3000, _png_data_url(0, 1024)), + ], +) +def test_image_estimate_uses_a_bounded_fallback_without_fetching_media( + model: str, detail: str, expected: int, data: str, +) -> None: + with patch("httpx.Client.send") as send: + estimate: Final = estimate_image_tokens(model, data, detail) + send.assert_not_called() + assert estimate == expected + assert estimate >= max( + estimate_image_tokens(model, _png_data_url(width, height), detail) + for width, height in ((16, 16), (1024, 1024), (4096, 4096), (768, 2000)) + ) + + +@pytest.mark.parametrize("mime_type", ["image/png", "application/octet-stream"]) +def test_image_estimate_bounds_decoding_independent_of_payload_size(mime_type: str) -> None: + header: Final = base64.b64decode(_png_data_url(1024, 1024).split(",", 1)[1]) + data: Final = f"data:{mime_type};base64," + base64.b64encode(header + b"\0" * 4_000_000).decode() + with patch("base64.b64decode", wraps=base64.b64decode) as decode_header: + estimate: Final = estimate_image_tokens(("gpt-6-astra", "azure/opaque"), data, "high", fallback_model="gpt-4o-mini") + assert estimate == estimate_image_tokens("gpt-4o-mini", _png_data_url(1024, 1024), "high") + assert decode_header.call_count == 1 + assert len(decode_header.call_args.args[0]) <= 65_536 + + +@pytest.mark.parametrize( + ("model", "canonical"), + [ + ("openai/gpt-4o-mini-2026-10-01", "gpt-4o-mini"), + ("azure/gpt-6-astra", "gpt-6-astra"), + ("azure/openai/gpt-4o-mini", "gpt-4o-mini"), + ("azure_ai/gpt-6-astra", "gpt-6-astra"), + ("azure_ai/openai/gpt-4o-mini", "gpt-4o-mini"), + ("openrouter/openai/gpt-6-astra", "gpt-6-astra"), + ("azure/responses/gpt-4o-mini", "gpt-4o-mini"), + ("azure/o_series/gpt-6-astra", "gpt-6-astra"), + ("azure/gpt-41", "gpt-4.1"), + ("azure/openai/gpt-41-mini", "gpt-4.1-mini"), + ("other/openai/gpt-4o", "unknown"), + ("azure/opaque/gpt-4o", "unknown"), + ("openai/azure/gpt-4o", "unknown"), + ("azure/openai/openai/gpt-4o", "unknown"), + ("openrouter/other/gpt-4o", "unknown"), + ("azure/gpt-4o-custom", "unknown"), + ], +) +def test_image_estimate_resolves_only_supported_model_identities(model: str, canonical: str) -> None: + data: Final = _png_data_url(1024, 1024) + assert estimate_image_tokens(model, data, "high") == estimate_image_tokens(canonical, data, "high") + + +def test_image_estimate_recognizes_cataloged_azure_names_without_caching_unknowns() -> None: + model: Final = "azure/image-policy-test-region/gpt-4o-mini" + data: Final = _png_data_url(1024, 1024) + assert estimate_image_tokens(model, data, "high") == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT + + litellm.register_model( + {model: {"litellm_provider": "azure", "mode": "chat", "input_cost_per_token": 0, "output_cost_per_token": 0}} + ) + + assert estimate_image_tokens(model, data, "high") == estimate_image_tokens("gpt-4o-mini", data, "high") + + +@pytest.mark.parametrize( + ("models", "fallback", "expected_model"), + [ + (("azure/deployment-prod",), "gpt-4o", "gpt-4o"), + (("openai/gpt-4o",), "gpt-4o-mini", "gpt-4o"), + (("azure/deployment-prod", "openai/gpt-4o"), "gpt-4o-mini", "gpt-4o-mini"), + (("openai/gpt-4o", "openai/gpt-4o-mini"), None, "gpt-4o-mini"), + ((), "gpt-6-astra", "gpt-6-astra"), + (("other/gpt-4o-mini",), "unregistered-model", "unregistered-model"), + (("azure/openai/gpt-4o",), "gpt-4o-mini", "gpt-4o"), + (("azure/openai/deployment-prod",), "azure_ai/openai/gpt-4o-mini", "gpt-4o-mini"), + (("azure/openai/gpt-4o", "azure/openai/deployment-prod"), "gpt-4o-mini", "gpt-4o-mini"), + ], +) +def test_image_estimate_prefers_each_backend_policy_over_the_requested_model( + models: tuple[str, ...], fallback: str | None, expected_model: str +) -> None: + data: Final = _png_data_url(1024, 1024) + assert estimate_image_tokens(models, data, "high", fallback_model=fallback) == estimate_image_tokens( + expected_model, data, "high" + ) + + +def test_image_estimate_falls_back_when_dimensions_are_beyond_header_budget() -> None: + jpeg: Final = ( + b"\xff\xd8\xff\xe1\xff\xff" + + b"\0" * 65533 + + b"\xff\xc0\x00\x0b\x08" + + (512).to_bytes(2, "big") + + (1024).to_bytes(2, "big") + ) + data: Final = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode() + assert estimate_image_tokens("gpt-4o", data, "high") == high_detail_image_token_upper_bound() + + HUB_SETUP_SCRIPT: Final = """ import json import sys diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 3d1e494e36c..a831415853f 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -15,19 +15,29 @@ Redis. """ import asyncio +import base64 +import io +import wave +from functools import partial import json from datetime import datetime, timedelta -from typing import Any, Dict, Final +from typing import Any, Dict, Final, Literal import pytest +from fastapi import HTTPException +from pydantic import TypeAdapter +from litellm import Router, token_counter from litellm.caching.caching import DualCache +from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT +from litellm.litellm_core_utils.audio_utils.utils import AUDIO_BYTES_PER_TOKEN from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, RateLimitedModel, - _AUDIO_BYTES_PER_TOKEN, + _resolve_media_models, PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( @@ -37,6 +47,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( get_request_stash, ) from litellm.proxy.utils import InternalUsageCache, hash_token +from litellm.router_utils.ptu_shares import PTUTeamCeiling from litellm.types.llms.openai import ( InputTokensDetails, ResponseAPIUsage, @@ -53,6 +64,397 @@ def rate_limiter(): return handler, cache +@pytest.mark.parametrize("output_cap,configured_output", [(None, None), (None, 32), (0, 32), (16, None)]) +def test_media_only_input_keeps_the_existing_output_budget( + rate_limiter: tuple[RateLimitHandler, DualCache], output_cap: int | None, configured_output: int | None, +) -> None: + handler, _ = rate_limiter + data: Final = { + "messages": [{"role": "user", "content": [{"type": "image", "source": {"type": "file", "file_id": "opaque"}}]}], + "n": 2, **({"max_tokens": output_cap} if output_cap is not None else {}), + } + input_tokens, output_tokens = handler._estimate_input_and_output_tokens( + data, call_type="acompletion", configured_output_tokens=configured_output, + ) + expected_output: Final = output_cap if output_cap is not None else configured_output or handler.no_max_tokens_output_floor(None) + assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT + assert output_tokens == expected_output * 2 + + +def _audio_message(seconds: int, sample_rate: int, channels: int) -> dict[str, object]: + output: Final = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(channels) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(b"\x00\x00" * seconds * sample_rate * channels) + return {"role": "user", "content": [{ + "type": "input_audio", "input_audio": { + "data": base64.b64encode(output.getvalue()).decode(), "format": "wav", + }, + }]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("detail,admitted", [("low", True), ("high", False)]) +async def test_model_specific_images_resolve_routes_before_admission( + detail: str, admitted: bool, equivalent_image_urls: tuple[str, str], +) -> None: + router: Final = Router( + model_list=[{ + "model_name": "vision", "litellm_params": { + "model": "azure/custom-deployment", "base_model": "gpt-4o-mini", + "api_key": "test", "api_base": "https://example.invalid", "api_version": "2024-12-01-preview", + }, "model_info": {"base_model": "gpt-6-astra"}, + }], model_group_alias={"alias": "vision"}, + ) + cache: Final = DualCache() + handler: Final = RateLimitHandler( + internal_usage_cache=InternalUsageCache(cache), media_model_resolver=partial(_resolve_media_models, router), + ) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-image-policy"), tpm_limit=500) + data: Final = {"model": "alias", "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": equivalent_image_urls[1], "detail": detail}}, + ]}], "max_tokens": 16} + if not admitted: + with pytest.raises(ProxyRateLimitError): + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + return + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + stash: Final = get_request_stash() + assert stash is not None + assert 250 < stash.reserved_tokens < 500 + + +@pytest.fixture(scope="module") +def equivalent_image_urls() -> tuple[str, str]: + from PIL import Image + + def encoded_image(compression: int) -> str: + output: Final = io.BytesIO() + Image.new("RGB", (1024, 1024)).save(output, format="PNG", compress_level=compression) + return "data:image/png;base64," + base64.b64encode(output.getvalue()).decode() + + return encoded_image(0), encoded_image(9) + + +def _structured_image_request( + image_url: str, representation: str, detail: str = "high" +) -> tuple[str, dict[str, object]]: + image: Final = {"type": "image_url", "image_url": {"url": image_url, "detail": detail}} + text: Final = {"type": "text", "text": "Describe this image"} + base: Final = {"model": "gpt-6-astra"} + if representation == "chat": + return "acompletion", {**base, "messages": [{"role": "user", "content": [text, image]}], "max_tokens": 16} + if representation == "chat_input_image": + input_image: Final = {**image, "type": "input_image"} + return "acompletion", {**base, "messages": [{"role": "user", "content": [text, input_image]}], "max_tokens": 16} + if representation == "anthropic_tool": + source_image: Final = {"type": "image", "source": { + "type": "base64", "media_type": "image/png", "data": image_url.partition(",")[2], + }} + return "anthropic_messages", {**base, "messages": [{"role": "user", "content": [{ + "type": "tool_result", "tool_use_id": "call_image", "content": [text, source_image], + }]}], "max_tokens": 16} + if representation == "document": + return "anthropic_messages", {**base, "messages": [{"role": "user", "content": [{ + "type": "document", "source": {"type": "content", "content": [text, image]}, + }]}], "max_tokens": 16} + response_content: Final = [ + {"type": "input_text", "text": "Describe this image"}, + {"type": "input_image", "image_url": image_url, "detail": detail}, + ] + response_input: Final = ( + [{"type": "custom_tool_call_output", "call_id": "call_image", "output": response_content}] + if representation == "custom_output" + else [{"role": "user", "content": response_content}] + ) + return "aresponses", {**base, "input": response_input, "max_output_tokens": 16} + + +@pytest.mark.parametrize("representation", ["chat", "chat_input_image", "anthropic_tool", "document", "responses", "custom_output"]) +def test_structured_media_estimates_ignore_transport_size_in_every_quota( + rate_limiter: tuple[RateLimitHandler, DualCache], equivalent_image_urls: tuple[str, str], representation: str, +) -> None: + handler, _ = rate_limiter + uncompressed, compressed = equivalent_image_urls + assert len(uncompressed) > 100 * len(compressed) + call_type, first = _structured_image_request(uncompressed, representation) + _, second = _structured_image_request(compressed, representation) + original: Final = json.dumps(first) + first_input: Final = handler._estimate_request_input_tokens(first, call_type=call_type, precise=True) + second_input: Final = handler._estimate_request_input_tokens(second, call_type=call_type, precise=True) + assert first_input == second_input + assert 500 < first_input.cheap_tokens < 5000 + assert first_input.precise_tokens is not None and 500 < first_input.precise_tokens < 5000 + assert handler._estimate_input_and_output_tokens(first, call_type=call_type) == ( + first_input.cheap_tokens, 16, + ) + assert json.dumps(first) == original + + +@pytest.mark.parametrize("tool_kind", ["function_call", "custom_tool_call"]) +def test_responses_tool_arguments_grow_reservations( + rate_limiter: tuple[RateLimitHandler, DualCache], + equivalent_image_urls: tuple[str, str], + tool_kind: str, +) -> None: + handler, _ = rate_limiter + output_request: Final = _structured_image_request(equivalent_image_urls[1], "custom_output")[1] + tool_output: Final = TypeAdapter(list[dict[str, object]]).validate_python(output_request["input"])[0] + requests: Final = tuple( + { + **output_request, + "input": [ + { + "type": tool_kind, "call_id": "call_image", "name": "inspect", + **( + {"arguments": json.dumps({"query": "a" * size})} + if tool_kind == "function_call" else {"input": "a" * size} + ), + }, + { + **tool_output, + "type": tool_kind + "_output", + }, + ], + } + for size in (8, 8008) + ) + estimates: Final = tuple( + handler._estimate_request_input_tokens(data, call_type="aresponses", precise=True) for data in requests + ) + assert estimates[1].cheap_tokens - estimates[0].cheap_tokens == 2000 + + chat_estimates: Final = tuple( + handler._estimate_request_input_tokens( + {"model": "gpt-6-astra", "messages": handler._responses_input_to_chat_messages(data)}, + call_type="acompletion", + ).cheap_tokens + for data in requests + ) + assert chat_estimates[0] == chat_estimates[1] + with_tools: Final = { + **requests[1], "tools": [{"type": "function", "name": "inspect", "description": "d" * 12000}] + } + assert ( + handler._estimate_request_input_tokens(with_tools, call_type="aresponses").cheap_tokens + == estimates[1].cheap_tokens + ) + + unsupported_content: Final = { + **requests[1], "input": [ + *TypeAdapter(list[dict[str, object]]).validate_python(requests[1]["input"]), + {"role": "user", "content": [ + {"type": "guarded_text", "text": ""}, + {"type": "input_file", "filename": "opaque.pdf", "file_data": "data:application/pdf;base64,eA=="}, + ]}, + ], + } + with pytest.raises(ValueError, match="guarded_text"): + token_counter( + model="gpt-6-astra", messages=handler._responses_input_to_chat_messages(unsupported_content), + use_default_image_token_count=True, + ) + fallback: Final = handler._estimate_request_input_tokens(unsupported_content, call_type="aresponses", precise=True) + assert fallback.precise_tokens == fallback.cheap_tokens == estimates[1].cheap_tokens + DEFAULT_IMAGE_TOKEN_COUNT + + +@pytest.mark.parametrize("representation", ["chat", "responses", "anthropic_tool", "document_content"]) +def test_opaque_documents_keep_a_cheap_reservation_and_existing_precise_count( + rate_limiter: tuple[RateLimitHandler, DualCache], representation: str, +) -> None: + from pypdf import PdfWriter + + handler, _ = rate_limiter + writer: Final = PdfWriter() + writer.add_blank_page(width=612, height=792) + writer.add_blank_page(width=612, height=792) + output: Final = io.BytesIO() + writer.write(output) + encoded: Final = base64.b64encode(output.getvalue()).decode() + file: Final = {"type": "file", "file": {"filename": "document.pdf", "file_data": "data:application/pdf;base64," + encoded}} + document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": encoded}} + content: Final = ( + [{"type": "tool_result", "tool_use_id": "read", "content": [document, document]}] + if representation == "anthropic_tool" + else [{"type": "document", "source": {"type": "content", "content": [file, file]}}] + if representation == "document_content" + else [file, file] + ) + request: Final = ( + {"model": "gpt-6-astra", "input": [{"role": "user", "content": [ + {"type": "input_file", **file["file"]}, {"type": "input_file", **file["file"]}, + ]}], "max_output_tokens": 16} + if representation == "responses" + else {"model": "gpt-6-astra", "messages": [{"role": "user", "content": content}], "max_tokens": 16} + ) + call_type: Final = "aresponses" if representation == "responses" else "acompletion" + original: Final = json.dumps(request) + estimate: Final = handler._estimate_request_input_tokens(request, call_type=call_type, precise=True) + assert estimate.cheap_tokens == 2 * DEFAULT_IMAGE_TOKEN_COUNT + assert handler._estimate_input_and_output_tokens(request, call_type=call_type) == (estimate.cheap_tokens, 16) + messages: Final = handler._responses_input_to_chat_messages(request) if representation == "responses" else request["messages"] + assert estimate.precise_tokens == token_counter( + model="gpt-6-astra", messages=messages, use_default_image_token_count=True, + ) + assert estimate.precise_tokens > estimate.cheap_tokens + assert json.dumps(request) == original + for _ in range(18): + writer.add_blank_page(width=612, height=792) + large_output: Final = io.BytesIO() + writer.write(large_output) + large_encoded: Final = base64.b64encode(large_output.getvalue()).decode() + large_request: Final = TypeAdapter(dict[str, object]).validate_json(original.replace(encoded, large_encoded)) + large_estimate: Final = handler._estimate_request_input_tokens(large_request, call_type=call_type) + assert large_estimate.cheap_tokens > estimate.cheap_tokens + assert large_estimate.cheap_tokens >= len(large_encoded) // 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("quota", ["combined", "project", "ptu"]) +@pytest.mark.parametrize("detail,limit,admitted", [("low", 500, True), ("high", 500, False), ("high", 2000, True)]) +async def test_image_detail_controls_each_admission_path( + equivalent_image_urls: tuple[str, str], quota: str, detail: str, limit: int, admitted: bool, +) -> None: + def ceiling(_team: str, model: str) -> PTUTeamCeiling | None: + return PTUTeamCeiling(model_group=model, tpm_limit=limit, output_to_input_ratio=4.0, cached_input_ratio=0.0) if quota == "ptu" else None + + cache: Final = DualCache() + handler: Final = RateLimitHandler( + internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=ceiling, + ) + auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-detail-quota"), team_id="media-team", + tpm_limit=limit if quota == "combined" else None, project_id="media-project", + project_metadata={"model_itpm_limit": {"gpt-6-astra": limit}} if quota == "project" else {}, + ) + call_type, data = _structured_image_request(equivalent_image_urls[1], "chat", detail) + if admitted: + await handler.async_pre_call_hook(auth, cache, data, call_type) + stash: Final = get_request_stash() + assert stash is not None + assert max(stash.reserved_tokens, stash.itpm_reserved_tokens, stash.ptu_reserved_tokens) > 250 + return + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook(auth, cache, data, call_type) + assert error.value.status_code == (400 if quota == "ptu" else 429) + + +def test_precise_tokenizer_fallback_reuses_nested_media_and_visible_text_once( + rate_limiter: tuple[RateLimitHandler, DualCache], equivalent_image_urls: tuple[str, str], +) -> None: + handler, _ = rate_limiter + call_type, request = _structured_image_request(equivalent_image_urls[1], "anthropic_tool") + data: Final = {**request, "messages": [ + *TypeAdapter(list[object]).validate_python(request["messages"]), + _audio_message(31, 48000, 2), + {"role": "user", "content": [{"type": "unsupported_counter_block"}]}, + ]} + measured: Final = handler._estimate_request_input_tokens(data, call_type=call_type, precise=True) + image_only: Final = handler._estimate_request_input_tokens(request, call_type=call_type) + assert measured.precise_tokens == measured.cheap_tokens == image_only.cheap_tokens + 310 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "failure", "partial", "project_rejection"]) +async def test_mixed_media_reservations_settle_each_bucket_in_its_own_units( + equivalent_image_urls: tuple[str, str], outcome: Literal["success", "failure", "partial", "project_rejection"], +) -> None: + now: Final = datetime(2026, 1, 2, 3, 4, 5) + + def ceiling(_team: str, model: str) -> PTUTeamCeiling: + return PTUTeamCeiling(model_group=model, tpm_limit=10000, output_to_input_ratio=4.0, cached_input_ratio=0.0) + + cache: Final = DualCache() + handler: Final = RateLimitHandler( + internal_usage_cache=InternalUsageCache(cache), time_provider=lambda: now, ptu_team_ceiling_resolver=ceiling, + ) + auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-media-settlement"), tpm_limit=10000, team_id="media-team", project_id="media-project", + project_metadata={ + "model_itpm_limit": {"gpt-6-astra": 500 if outcome == "project_rejection" else 2000}, + "model_otpm_limit": {"gpt-6-astra": 10000}, + }, + ) + _, image_request = _structured_image_request(equivalent_image_urls[1], "chat") + data: Final = {**image_request, "messages": [*TypeAdapter(list[object]).validate_python(image_request["messages"]), _audio_message(31, 24000, 1)]} + scopes: Final = ( + ("api_key", auth.api_key), + ("model_per_team_ptu", "media-team:gpt-6-astra"), + (PROJECT_ITPM_DESCRIPTOR_KEY, "media-project:gpt-6-astra"), + (PROJECT_OTPM_DESCRIPTOR_KEY, "media-project:gpt-6-astra"), + ) + if outcome == "project_rejection": + with pytest.raises(ProxyRateLimitError): + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + else: + await handler.async_pre_call_hook(auth, cache, data, "acompletion") + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_tokens > 1500 + assert stash.itpm_reserved_tokens > 1500 + assert stash.ptu_reserved_tokens == stash.reserved_tokens + 3 * stash.otpm_reserved_tokens + usage: Final = Usage( + prompt_tokens=40, completion_tokens=10, total_tokens=50, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=15), + ) + kwargs: Final = { + "litellm_call_id": data.get("litellm_call_id"), + "model": "gpt-6-astra", + "standard_logging_object": {"metadata": { + "user_api_key_hash": auth.api_key, "user_api_key_team_id": auth.team_id, + "user_api_key_project_id": auth.project_id, + }}, + **({"combined_usage_object": usage} if outcome == "partial" else {}), + } + if outcome == "success": + response: Final = ModelResponse(model="gpt-6-astra", usage=usage, choices=[]) + await handler.async_log_success_event(kwargs, response, now, now) + else: + await handler.async_log_failure_event(kwargs, None, now, now) + await handler.async_log_failure_event(kwargs, None, now, now) + expected: Final = (35, 65, 25, 10) if outcome in ("success", "partial") else (0, 0, 0, 0) + for (scope, value), amount in zip(scopes, expected): + key: Final = handler.create_rate_limit_keys(scope, value or "", "tokens") + assert int(await cache.async_get_cache(key=key, local_only=True) or 0) == amount + + +@pytest.mark.asyncio +@pytest.mark.parametrize("representation", ["chat", "inlineData"]) +@pytest.mark.parametrize("seconds,admitted", [(1, True), (31, False)]) +async def test_native_and_chat_audio_share_duration_based_quota_admission( + rate_limiter: tuple[RateLimitHandler, DualCache], representation: str, seconds: int, admitted: bool, +) -> None: + handler, cache = rate_limiter + model: Final = "gemini/gemini-3.8-flash" + message: Final = _audio_message(seconds, 24000, 1) + blocks: Final = TypeAdapter(list[dict[str, object]]).validate_python(message["content"]) + audio: Final = TypeAdapter(dict[str, str]).validate_python(blocks[0]["input_audio"]) + native: Final = representation != "chat" + data: Final = ( + {"model": model, "contents": [{"role": "user", "parts": [{ + "inlineData": {"mimeType": "audio/wav", "data": audio["data"]}, + }]}], "config": {"maxOutputTokens": 16}} + if native + else {"model": model, "messages": [message], "max_tokens": 16} + ) + auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-native-duration"), tpm_limit=100, project_id="audio-project", + project_metadata={"model_itpm_limit": {model: 100}}, + ) + call_type: Final = "agenerate_content" if native else "acompletion" + if admitted: + await handler.async_pre_call_hook(auth, cache, data, call_type) + stash: Final = get_request_stash() + assert stash is not None + assert 26 < stash.reserved_tokens < 100 + assert 10 < stash.itpm_reserved_tokens < 100 + return + with pytest.raises(ProxyRateLimitError): + await handler.async_pre_call_hook(auth, cache, data, call_type) + + @pytest.fixture(autouse=True) def _isolated_request_stash(): token = _request_stash.set(None) @@ -2274,7 +2676,7 @@ def test_audio_token_estimate_scales_with_payload_size(): to exhaust ITPM quota while reserving almost nothing. The estimate must now grow proportionally with the base64 payload size - (len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN), floored at + (len(b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN), floored at DEFAULT_AUDIO_TOKEN_ESTIMATE so reference-only blocks and genuinely short clips still get a non-trivial reservation. @@ -2312,7 +2714,7 @@ def test_audio_token_estimate_scales_with_payload_size(): f"Large payload ({large_estimate}) must reserve more than small payload " f"({small_estimate}); flat-rate bug is back" ) - assert very_large_estimate == len(very_large_b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN + assert very_large_estimate == len(very_large_b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN assert very_large_estimate > 6_000 assert no_data_estimate >= 300, ( f"Reference-only block (no data) must use the DEFAULT_AUDIO_TOKEN_ESTIMATE floor; got {no_data_estimate}" @@ -2792,26 +3194,6 @@ async def test_reserve_io_tokens_itpm_only_no_otpm(rate_limiter): assert otpm_reserved == 0 -def test_strip_audio_content_blocks_passthrough_non_list_messages(): - """Non-list input is returned unchanged (early return on line 2605).""" - result = RateLimitHandler._strip_audio_content_blocks("not a list") - assert result == "not a list" - - -def test_strip_audio_content_blocks_passthrough_non_dict_message(): - """Non-dict entries in the message list are appended unchanged.""" - messages = ["plain string message"] - result = RateLimitHandler._strip_audio_content_blocks(messages) - assert result == ["plain string message"] - - -def test_strip_audio_content_blocks_passthrough_non_list_content(): - """Messages with non-list content (e.g. plain string) pass through unchanged.""" - messages = [{"role": "user", "content": "hello"}] - result = RateLimitHandler._strip_audio_content_blocks(messages) - assert result == [{"role": "user", "content": "hello"}] - - @pytest.mark.asyncio async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter): """ @@ -3519,6 +3901,28 @@ def test_split_token_estimate_selects_endpoint_input(rate_limiter, call_type, da assert estimated_output == 1 +@pytest.mark.parametrize( + "call_type,data,cheap_input,precise_input", + [ + ("embedding", {"input": [1, 2, 3]}, 1, 3), + ("embedding", {"input": [[1, 2], [3]]}, 2, 3), + ("rerank", {"query": "query text", "documents": ["first document", "second document"]}, 0, None), + ], +) +def test_nonmedia_input_keeps_distinct_cheap_and_precise_reservations( + rate_limiter: tuple[RateLimitHandler, DualCache], call_type: str, data: dict[str, object], + cheap_input: int, precise_input: int | None, +) -> None: + handler, _ = rate_limiter + estimate: Final = handler._estimate_request_input_tokens(data, "gpt-6-astra", call_type, precise=True) + assert handler._estimate_input_and_output_tokens( + data, min_configured_tpm_limit=100, call_type=call_type, input_estimate=estimate, + ) == (cheap_input, 0) + assert estimate.precise_tokens is not None and estimate.precise_tokens > cheap_input + if precise_input is not None: + assert estimate.precise_tokens == precise_input + + def test_split_quota_multimodal_guards_handle_non_mapping_inputs(rate_limiter): handler, _cache = rate_limiter diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 568a05e8ff4..a19a839c1f6 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20261,8 +20261,9 @@ def _resolution_router() -> Router: ], ids=["exact-name", "model-group-alias", "every-member-of-a-pool", "wildcard-expands", "resolves-to-nothing"], ) +@pytest.mark.parametrize("use_base_model", (False, True)) def test_resolved_litellm_models_answers_through_every_channel_a_request_uses( - model_name: str, expected: tuple[str, ...] + model_name: str, expected: tuple[str, ...], use_base_model: bool ) -> None: """A caller comparing two names by what serves them needs each channel the request path composes, since the deployment name an admin picked carries no information on its own. @@ -20270,7 +20271,7 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses( `resolves-to-nothing` is the contract that keeps the fallback out of here: an empty result is not "the call fails", so what to do about it stays each caller's policy. """ - assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected) + assert set(_resolution_router().resolved_litellm_models(model_name, use_base_model=use_base_model)) == set(expected) class TestTierParamsTheTargetAccepts: @@ -30149,3 +30150,67 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items(): assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False + + +@pytest.mark.parametrize("info_base,params_base", [(None, None), (None, "gpt-5.1"), ("gpt-6-astra", "gpt-5.1")]) +def test_resolved_litellm_models_can_use_configured_base_model( + info_base: str | None, params_base: str | None, +) -> None: + router: Final = Router( + model_list=[{ + "model_name": "deployment", "litellm_params": { + "model": "azure/custom-name", "api_key": "test", "api_base": "https://example.invalid", + "api_version": "2024-12-01-preview", **({"base_model": params_base} if params_base else {}), + }, "model_info": {"base_model": info_base}, "rpm": 10.5, + }], model_group_alias={"alias": "deployment"}, + ) + assert router.resolved_litellm_models("alias") == ("azure/custom-name",) + assert router.resolved_litellm_models("alias", use_base_model=True) == ( + info_base or params_base or "azure/custom-name", + ) + + +def test_resolved_model_identity_does_not_revalidate_stored_quotas() -> None: + router: Final = Router( + model_list=[{ + "model_name": "vision/*", + "litellm_params": { + "model": "azure/vision-deployment", "api_key": "test", "api_base": "https://example.invalid", + "api_version": "2024-12-01-preview", "base_model": "gpt-5.1", + }, + "model_info": {"base_model": "gpt-6-astra"}, + "tpm": 10.5, + }], + ignore_invalid_deployments=False, + ) + + assert router.resolved_litellm_models("vision/image") == ("azure/vision-deployment",) + assert router.resolved_litellm_models("vision/image", use_base_model=True) == ("gpt-6-astra",) + + +@pytest.mark.parametrize("explicit_provider", (False, True)) +@pytest.mark.parametrize("profile_known", (False, True)) +def test_resolved_model_identity_uses_only_mapped_bedrock_profiles( + monkeypatch: pytest.MonkeyPatch, explicit_provider: bool, profile_known: bool, +) -> None: + profile: Final = "us.anthropic.identity-test" + model: Final = "converse/anthropic.identity-test" + monkeypatch.setattr(litellm, "model_cost", { + **litellm.model_cost, + **({profile: { + "litellm_provider": "bedrock", "mode": "chat", + "input_cost_per_token": 0.01, "output_cost_per_token": 0.02, + }} if profile_known else {}), + }) + configured_model: Final = model if explicit_provider else f"bedrock/{model}" + router: Final = Router(model_list=[{ + "model_name": "profile", "litellm_params": { + "model": configured_model, "model_id": profile, + **({"custom_llm_provider": "bedrock"} if explicit_provider else {}), + }, + }]) + + assert router.resolved_litellm_models("profile") == (configured_model,) + assert router.resolved_litellm_models("profile", use_base_model=True) == ( + f"bedrock/{profile}" if profile_known else configured_model, + )