This commit is contained in:
tin-berri 2026-10-27 01:31:46 +00:00 • committed by GitHub
commit d5899eca09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1565 additions and 254 deletions

View file

@ -2,8 +2,12 @@
Utils used for litellm.transcription() and litellm.atranscription()
"""
import base64
import binascii
import hashlib
import math
import os
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Final
@ -331,6 +335,44 @@ def calculate_request_duration(file: FileTypes) -> float | None:
return None
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
AUDIO_TOKENS_PER_SECOND: Final = 10
AUDIO_BYTES_PER_TOKEN: Final = 1600
_MAX_AUDIO_ESTIMATE_BYTES: Final = 16 * 1024 * 1024
def _audio_token_rate(model: str) -> int | None:
if model.startswith(
("gemini-", "gemini/gemini-", "vertex_ai/gemini-", "vertex_ai_beta/gemini-", "openrouter/google/gemini-")
):
return 32
if model.startswith(("gpt-", "openai/gpt-", "azure/gpt-", "openrouter/openai/gpt-")):
return AUDIO_TOKENS_PER_SECOND
return None
def estimate_audio_tokens(data: str | None, models: Sequence[str] = (), fallback_model: str | None = None) -> int:
"""Estimate audio tokens from container duration within a bounded decoding budget."""
if not data:
return DEFAULT_AUDIO_TOKEN_ESTIMATE
fallback_estimate: Final = max(len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
if len(data) * 3 // 4 - padding > _MAX_AUDIO_ESTIMATE_BYTES:
return fallback_estimate
try:
duration: Final = calculate_request_duration(base64.b64decode(data, validate=True))
except (ValueError, binascii.Error):
return fallback_estimate
if duration is None or not math.isfinite(duration) or duration <= 0:
return fallback_estimate
fallback_rate: Final = _audio_token_rate(fallback_model) if fallback_model is not None else None
rate: Final = max(
(_audio_token_rate(model) or fallback_rate or AUDIO_TOKENS_PER_SECOND for model in models),
default=fallback_rate or AUDIO_TOKENS_PER_SECOND,
)
return max(1, math.ceil(duration * rate))
DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"

View file

@ -9,6 +9,7 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
@ -234,6 +235,199 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
_IMAGE_SCAN_MAX_DEPTH: Final = 4
_MEDIA_CONTENT_PART_TYPES: Final = _IMAGE_CONTENT_PART_TYPES | {"input_audio"}
_REMOVED_MEDIA_PART: Final = object()
@dataclass(frozen=True, slots=True)
class _ContentMediaSplit:
content: object
media: tuple[Mapping[str, object], ...] = ()
text: str = ""
opaque_document_sizes: tuple[int, ...] = ()
def _message_content_parts(value: object) -> Sequence[object] | None:
if not isinstance(value, (list, tuple)):
return None
return cast(Sequence[object], value) # cast-ok: finite JSON content arrays
def _file_media_part(part: Mapping[str, object]) -> Mapping[str, object] | None:
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
file: Final = as_str_mapping(part.get("file")) if part.get("type") == "file" else part
if file is None:
return None
data: Final = file.get("file_data")
data_string: Final = data if isinstance(data, str) else ""
declared_type: Final = next(
(value for key in ("format", "mime_type", "content_type") if isinstance(value := file.get(key), str) and value),
"",
)
mime_type: Final = (declared_type or parse_mime_type(data_string[:256]) or "").split(";", 1)[0].strip().lower()
if mime_type.startswith("image/"):
file_id: Final = file.get("file_id")
url: Final = data_string or (file_id if isinstance(file_id, str) else "")
image_url: Final = (
url
if not data_string or url.startswith(("data:", "https://", "http://", "gs://"))
else f"data:{mime_type};base64,{url}"
)
return {"type": "image_url", "image_url": {"url": image_url, "detail": file.get("detail", "auto")}}
if mime_type.startswith("audio/"):
header_end: Final = data_string.find(",", 0, 256)
encoded: Final = (
data_string[header_end + 1 :]
if header_end >= 0 and data_string[:header_end].endswith(";base64")
else ""
if data_string.startswith(("data:", "https://", "http://", "gs://"))
else data_string
)
return {"type": "input_audio", "input_audio": {"data": encoded, "format": mime_type.removeprefix("audio/")}}
return None
def _opaque_document_size(payload: object) -> int:
fields: Final = as_str_mapping(payload)
strings: Final = {key: value for key, value in (fields or {}).items() if isinstance(value, str)}
return len(str(strings))
def _split_media_leaf(part: object) -> _ContentMediaSplit:
if isinstance(part, str):
return _ContentMediaSplit(part, text=part)
block: Final = as_str_mapping(part)
if block is None:
return _ContentMediaSplit(part)
part_type: Final = block.get("type")
media: Final = (
block
if isinstance(part_type, str) and part_type in _MEDIA_CONTENT_PART_TYPES
else _file_media_part(block)
if part_type in ("file", "input_file")
else None
)
if media is not None:
return _ContentMediaSplit(_REMOVED_MEDIA_PART, (media,))
text: Final = block.get("text")
return _ContentMediaSplit(
part,
text=text if isinstance(text, str) else "",
opaque_document_sizes=(
(_opaque_document_size(block.get("file") if part_type == "file" else block),)
if part_type in ("file", "input_file")
else ()
),
)
def _split_media_content(content: object, split_part: Callable[[object], _ContentMediaSplit]) -> _ContentMediaSplit:
parts: Final = _message_content_parts(content)
if parts is None:
return _ContentMediaSplit(content, text=content if isinstance(content, str) else "")
splits: Final = tuple(map(split_part, parts))
media: Final = tuple(chain.from_iterable(split.media for split in splits))
kept: Final = tuple(split.content for split in splits if split.content is not _REMOVED_MEDIA_PART)
sanitized: Final = content if not media else kept if isinstance(content, tuple) else list(kept)
return _ContentMediaSplit(
sanitized,
media,
"".join(split.text for split in splits),
tuple(chain.from_iterable(split.opaque_document_sizes for split in splits)),
)
def _split_document_media(part: object) -> _ContentMediaSplit:
block: Final = as_str_mapping(part)
if block is None or block.get("type") != "document":
return _split_media_leaf(part)
source: Final = as_str_mapping(block.get("source"))
metadata: Final = "".join(value for key in ("title", "context") if isinstance(value := block.get(key), str))
if source is None:
return _ContentMediaSplit(part, text=metadata, opaque_document_sizes=(0,))
if source.get("type") == "text":
text: Final = source.get("data")
return _ContentMediaSplit(part, text=metadata + (text if isinstance(text, str) else ""))
if source.get("type") != "content":
return _ContentMediaSplit(part, text=metadata, opaque_document_sizes=(_opaque_document_size(source),))
split: Final = _split_media_content(source.get("content"), _split_media_leaf)
sanitized: Final = {**block, "source": {**source, "content": split.content}} if split.media else part
return _ContentMediaSplit(sanitized, split.media, metadata + split.text, split.opaque_document_sizes)
def _split_tool_result_media(part: object) -> _ContentMediaSplit:
block: Final = as_str_mapping(part)
if block is None or block.get("type") != "tool_result":
return _split_document_media(part)
split: Final = _split_media_content(block.get("content"), _split_document_media)
sanitized: Final = {**block, "content": split.content} if split.media else part
return _ContentMediaSplit(sanitized, split.media, split.text, split.opaque_document_sizes)
def _split_message_content(message: object) -> _ContentMediaSplit:
fields: Final = as_str_mapping(message)
if fields is None:
return _ContentMediaSplit(message)
split: Final = _split_media_content(fields.get("content"), _split_tool_result_media)
sanitized: Final = {**fields, "content": split.content} if split.media else message
return _ContentMediaSplit(sanitized, split.media, split.text, split.opaque_document_sizes)
def split_message_media(messages: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
"""Split declared media from content without mutation, following at most tool-result and document nesting."""
parts: Final = _message_content_parts(messages)
if parts is None:
return messages, ()
splits: Final = tuple(map(_split_message_content, parts))
media: Final = tuple(chain.from_iterable(split.media for split in splits))
if not media:
return messages, ()
sanitized: Final = tuple(split.content for split in splits)
return (sanitized if isinstance(messages, tuple) else list(sanitized)), media
def _message_tool_argument_text(message: Mapping[str, object]) -> str:
if message.get("role") != "assistant":
return ""
functions: Final = chain(
(message.get("function_call"),),
(
call_fields.get("function")
for call in _message_content_parts(message.get("tool_calls")) or ()
if (call_fields := as_str_mapping(call)) is not None
),
)
return "".join(
arguments
for function in functions
if (function_fields := as_str_mapping(function)) is not None
and isinstance(arguments := function_fields.get("arguments"), str)
)
def get_token_count_text_from_messages(messages: object, *, include_tool_arguments: bool = False) -> str:
"""Visible message content, excluding media bytes and opaque document/reasoning payloads."""
text, _ = get_token_count_text_and_document_sizes(messages, include_tool_arguments=include_tool_arguments)
return text
def get_token_count_text_and_document_sizes(
messages: object, *, include_tool_arguments: bool = False
) -> tuple[str, tuple[int, ...]]:
"""Visible text and serialized opaque payload sizes, without decoding or fetching documents."""
fields: Final = tuple(
message for value in _message_content_parts(messages) or () if (message := as_str_mapping(value)) is not None
)
splits: Final = tuple(_split_message_content(message) for message in fields)
text: Final = "".join(
split.text
+ extract_search_results_text(message.get("search_results"))
+ (_message_tool_argument_text(message) if include_tool_arguments else "")
for message, split in zip(fields, splits)
)
return text, tuple(chain.from_iterable(split.opaque_document_sizes for split in splits))
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans

View file

@ -3,15 +3,18 @@
import base64
import io
import math
import re
import struct
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from typing_extensions import ParamSpec, TypeVar
import litellm
@ -35,6 +38,7 @@ from litellm.constants import (
TOKEN_COUNTER_MAX_EXACT_CHARS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
ChatTemplateToolSection,
chat_template_tool_section,
@ -59,7 +63,6 @@ from litellm.types.llms.anthropic import (
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionDocumentObject,
ChatCompletionNamedToolChoiceParam,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolParam,
@ -184,22 +187,22 @@ def resize_image_high_res(
# Test the function with the given example
def calculate_tiles_needed(
resized_width,
resized_height,
tile_width=MAX_TILE_WIDTH,
tile_height=MAX_TILE_HEIGHT,
):
resized_width: int,
resized_height: int,
tile_width: int = MAX_TILE_WIDTH,
tile_height: int = MAX_TILE_HEIGHT,
) -> int:
tiles_across: Final = (resized_width + tile_width - 1) // tile_width
tiles_down: Final = (resized_height + tile_height - 1) // tile_height
total_tiles: Final = tiles_across * tiles_down
return total_tiles
def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int:
def high_detail_image_token_upper_bound(base_tokens: int = 85, tile_tokens: int | None = None) -> int:
largest_tile_count: Final = calculate_tiles_needed(
MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES
)
return base_tokens + (base_tokens * 2) * largest_tile_count
return base_tokens + (base_tokens * 2 if tile_tokens is None else tile_tokens) * largest_tile_count
def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]:
@ -261,6 +264,10 @@ def get_image_dimensions(
_header, encoded = data.split(",", 1)
img_data = base64.b64decode(encoded)
return _get_image_dimensions_from_bytes(img_data) or (DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT)
def _get_image_dimensions_from_bytes(img_data: bytes) -> tuple[int, int] | None:
img_type: Final = get_image_type(img_data)
if img_type == "png":
@ -303,8 +310,7 @@ def get_image_dimensions(
h = ((bits >> 14) & 0x3FFF) + 1
return w, h
# return sensible default image dimensions if unable to get dimensions
return DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT
return None
def calculate_img_tokens(
@ -342,6 +348,161 @@ def calculate_img_tokens(
return total_tokens
_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
_MAX_IMAGE_PATCHES: Final = 30_000
_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
{
"gpt-4o": (85, 170),
"gpt-4.1": (85, 170),
"gpt-4o-mini": (2833, 5667),
"gpt-5": (70, 140),
"gpt-5.1": (70, 140),
"o1": (75, 150),
"o1-pro": (75, 150),
"o3": (75, 150),
}
)
_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
{
"gpt-6-astra": 1.2,
"gpt-5.6-sol": 1.2,
"gpt-5.6-terra": 1.2,
"gpt-5.6-luna": 1.2,
"gpt-5.5": 1.2,
"gpt-5.4": 1.2,
"gpt-5.4-mini": 1.2,
"gpt-5.4-nano": 1.2,
"gpt-5.2": 1.2,
"gpt-4.1-mini": 1.62,
}
)
def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
separator: Final = data.find(",", 0, 256)
# Adapters can declare the image MIME separately from a generic data URL.
# The bounded byte parser still requires a supported image signature.
if not data.startswith("data:") or separator < 0 or not data[:separator].endswith(";base64"):
return None
encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
try:
header: Final = base64.b64decode(encoded_header, validate=True)
dimensions: Final = _get_image_dimensions_from_bytes(header)
except (ValueError, TypeError, struct.error):
return None
if dimensions is None:
return None
width, height = dimensions
return (width, height) if width > 0 and height > 0 else None
def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
if model in ("gpt-5.2", "gpt-4.1-mini"):
return None if detail == "original" else (2048, 6144)
original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
if effective_detail == "low":
return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
if effective_detail == "high":
return (65_535 if model == "gpt-6-astra" else 2048, 2500)
return (
(65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
)
def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
dimension_scale: Final = min(1.0, max_dimension / max(width, height))
fitted_width: Final = max(1, round(width * dimension_scale))
fitted_height: Final = max(1, round(height * dimension_scale))
patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
if budget is None or patches <= budget:
return patches
shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
width_patches: Final = fitted_width * shrink / 32
height_patches: Final = fitted_height * shrink / 32
aligned_shrink: Final = shrink * min(
math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
)
resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
def _image_policy_model_name(model: str) -> str | None:
provider, separator, provider_model = model.partition("/")
if not separator:
return model
if provider == "openai":
return provider_model
if provider == "openrouter":
return provider_model.removeprefix("openai/") if provider_model.startswith("openai/") else None
if provider not in ("azure", "azure_ai"):
return None
catalog_entry: Final = as_str_mapping(
cast( # cast-ok: read-only view of the legacy string-keyed registry; as_str_mapping checks the entry
Mapping[str, object], litellm.model_cost
).get(model)
)
if catalog_entry is not None and catalog_entry.get("litellm_provider") in ("azure", "azure_ai"):
return provider_model.rsplit("/", 1)[-1]
return provider_model.removeprefix("responses/").removeprefix("o_series/").removeprefix("openai/")
def _image_model_family(model: str) -> str | None:
provider_model: Final = _image_policy_model_name(model)
if provider_model is None:
return None
family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
canonical: Final = litellm.azure_llms.get(f"azure/{family}", family) if model.startswith("azure/") else family
return canonical if canonical in _TILE_IMAGE_TOKEN_COSTS or canonical in _PATCH_IMAGE_MULTIPLIERS else None
def _image_policy_tokens(family: str | None, dimensions: tuple[int, int] | None, detail: str) -> int:
if family is None:
return DEFAULT_IMAGE_TOKEN_COUNT
tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
if tile_costs is not None:
base_tokens, tile_tokens = tile_costs
if detail == "original":
return DEFAULT_IMAGE_TOKEN_COUNT
if detail == "low":
return base_tokens
if dimensions is None:
return high_detail_image_token_upper_bound(base_tokens, tile_tokens)
width, height = dimensions
scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
tiles: Final = calculate_tiles_needed(max(1, math.floor(width * scale)), max(1, math.floor(height * scale)))
return base_tokens + tiles * tile_tokens
limits: Final = _patch_image_limits(family, detail)
if limits is None or multiplier is None:
return DEFAULT_IMAGE_TOKEN_COUNT
max_dimension, patch_budget = limits
patches: Final = (
min(math.ceil(max_dimension / 32) ** 2, patch_budget if patch_budget is not None else _MAX_IMAGE_PATCHES)
if dimensions is None
else _image_patch_count(*dimensions, max_dimension, patch_budget)
)
return math.ceil(patches * multiplier)
def estimate_image_tokens(
model: str | Sequence[str], data: str, detail: str = "auto", *, fallback_model: str | None = None
) -> int:
"""Estimate image policies from at most 48 KiB of inline headers, without fetching media."""
if detail not in ("low", "high", "auto", "original"):
return DEFAULT_IMAGE_TOKEN_COUNT
models: Final = (model,) if isinstance(model, str) else model
fallback_family: Final = _image_model_family(fallback_model) if fallback_model is not None else None
families: Final = tuple(
dict.fromkeys(_image_model_family(candidate) or fallback_family for candidate in models or ("",))
)
if all(family is None for family in families):
return DEFAULT_IMAGE_TOKEN_COUNT
dimensions: Final = _inline_image_dimensions(data)
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
TokenCounterFunction = Callable[[str], int]
"""
Type for a function that counts tokens in a string.
@ -351,6 +512,7 @@ EXTRAPOLATION_SAMPLES: Final = 16
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
T_Retval = TypeVar("T_Retval")
_COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
_TOOL_CHOICE_FUNCTION: Final = TypeAdapter(Mapping[str, object])
_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
)
@ -418,10 +580,10 @@ def token_counter(
model="",
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
text: str | list[str] | None = None,
messages: Sequence[AllMessageValues | Message] | None = None,
messages: Sequence[object] | None = None,
count_response_tokens: bool | None = False,
tools: list[ChatCompletionToolParam] | None = None,
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
tools: Sequence[object] | None = None,
tool_choice: object = None,
use_default_image_token_count: bool | None = False,
default_token_count: int | None = None,
) -> int:
@ -600,8 +762,8 @@ def _count_messages(
def _count_extra(
count_function: TokenCounterFunction,
tools: list[ChatCompletionToolParam] | None,
tool_choice: ChatCompletionNamedToolChoiceParam | None,
tools: Sequence[object] | None,
tool_choice: object,
includes_system_message: bool,
tool_section: ChatTemplateToolSection | None,
) -> int:
@ -638,12 +800,13 @@ def _count_tool_definitions(
def _count_tool_choice(
count_function: TokenCounterFunction,
tool_choice: ChatCompletionNamedToolChoiceParam | None,
tool_choice: object,
) -> int:
if tool_choice == "none":
return 1
if isinstance(tool_choice, dict):
return 7 + count_function(str(tool_choice["function"]["name"]))
function: Final = _TOOL_CHOICE_FUNCTION.validate_python(tool_choice["function"])
return 7 + count_function(str(function["name"]))
return 0

View file

@ -42,14 +42,22 @@ from litellm.caching.redis_batch import (
active_request_redis_batch,
)
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.constants import (
DEFAULT_IMAGE_TOKEN_COUNT,
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
as_openai_image_part,
get_token_count_text_and_document_sizes,
get_token_count_text_from_messages,
split_message_media,
)
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
from litellm.llms.azure.ptu_capacity import normalized_tokens
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
@ -73,7 +81,6 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
canonical_provider_batch_id,
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
from litellm.router_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
@ -98,6 +105,7 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.router import Router
from litellm.types.agents import AgentResponse
from litellm.types.caching import RedisPipelineIncrementOperation
@ -109,6 +117,9 @@ else:
_REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object])
_TOKEN_COUNT_ITEMS: Final = TypeAdapter(list[object])
_TOKEN_COUNT_SEQUENCE: Final = TypeAdapter[Sequence[object] | None](Sequence[object] | None)
_TOKEN_COUNT_TEXT: Final = TypeAdapter[str | list[str] | None](str | list[str] | None)
_MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None)
@ -135,6 +146,24 @@ class RateLimitedModel:
return requested_limit if requested_limit is not None else limits.get(self.group)
@dataclass(frozen=True, slots=True)
class RequestInputTokenEstimate:
cheap_tokens: int
precise_tokens: int | None = None
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
if router is None:
return (model,)
return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,)
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
from litellm.proxy.proxy_server import llm_router
return _resolve_media_models(llm_router, model, team_id)
def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
from litellm.proxy.proxy_server import llm_router
@ -502,18 +531,6 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
CallTypes.agenerate_content_stream.value,
)
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
# litellm.token_counter has no per-type handling for "input_audio" content
# blocks (unlike images, which use use_default_image_token_count) -- it
# silently contributes 0 tokens for them. When the block carries a base64
# payload, the estimate is derived from the decoded byte count; when the
# block is a reference without a payload (or the payload is missing), this
# flat per-block floor is used instead.
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
# Conservative bytes-per-token assumption for size-based audio estimation:
# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing
# the lowest reasonable bitrate means we never under-reserve for higher-
# quality audio recorded at the same wall-clock duration.
_AUDIO_BYTES_PER_TOKEN: Final = 1600
# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from
# "model_per_project" (the combined-TPM descriptor) so both can be enforced
# on the same project+model simultaneously without colliding on cache keys.
@ -867,11 +884,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
] = _resolve_ptu_team_ceiling_via_proxy_router,
fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
):
self.internal_usage_cache = internal_usage_cache
self._time_provider = time_provider or datetime.now
self._tag_rate_limit_resolver = tag_rate_limit_resolver
self._model_group_resolver = model_group_resolver
self._media_model_resolver = media_model_resolver
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
self._fail_closed_resolver = fail_closed_resolver
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
@ -1138,6 +1157,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit: int | None = None,
call_type: str | None = None,
configured_output_tokens: int | None = None,
input_estimate: RequestInputTokenEstimate | None = None,
) -> int:
"""
Estimate total tokens this request will consume so we can reserve them
@ -1159,11 +1179,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
floor entirely, so the reservation reflects what this tenant's model
actually emits rather than one constant shared by every tenant.
"""
measured_input: Final = (
input_estimate
if input_estimate is not None
else self._estimate_request_input_tokens(_REQUEST_RATE_LIMIT_DATA.validate_python(data), model, call_type)
)
estimated_input_tokens, max_tokens_estimate = self._estimate_input_and_output_tokens(
data=data,
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
input_estimate=measured_input,
)
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
@ -1184,6 +1210,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type: str | None,
configured_output_tokens: int | None,
raw_estimate: int,
input_estimate: RequestInputTokenEstimate | None = None,
) -> int:
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
@ -1194,6 +1221,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
input_estimate=input_estimate,
)
normalized: Final = normalized_tokens(
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
@ -1206,6 +1234,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit: int | None = None,
call_type: str | None = None,
configured_output_tokens: int | None = None,
input_estimate: RequestInputTokenEstimate | None = None,
) -> tuple[int, int]:
"""
Estimate input tokens and output (max_tokens) budget separately, so
@ -1231,43 +1260,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
"""
if not isinstance(data, dict):
return 0, 0
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
estimable_data: Final = translated_data if translated_data is not None else data
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
(None, None, estimable_data.get("input"))
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
else (None, estimable_data.get("prompt"), None)
if call_type in TEXT_COMPLETION_API_CALL_TYPES
else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(estimable_data)), None)
if call_type in EXTRACTION_API_CALL_TYPES
else (estimable_data.get("messages"), None, None)
if call_type
else (
estimable_data.get("messages"),
estimable_data.get("prompt"),
estimable_data.get("input"),
estimated_input_tokens: Final = (
input_estimate
if input_estimate is not None
else self._estimate_request_input_tokens(
_REQUEST_RATE_LIMIT_DATA.validate_python(data), call_type=call_type
)
)
messages, prompt, raw_input = selected_fields
input_text: Final = (
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
)
total_chars: Final = (
len(get_str_from_messages(messages))
if isinstance(messages, list) and messages
else len(prompt)
if isinstance(prompt, str)
else sum(len(str(item)) for item in prompt)
if isinstance(prompt, list)
else len(input_text)
if isinstance(input_text, str)
else sum(len(str(item)) for item in input_text)
if isinstance(input_text, list)
else 0
)
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
).cheap_tokens
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
generates_no_output: Final = (
@ -1283,7 +1282,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
max_tokens_estimate: Final = (
0
if generates_no_output
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
else explicit_max_tokens
if explicit_max_tokens is not None
else configured_output_tokens
@ -3753,72 +3752,37 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
@staticmethod
def _estimate_audio_block_tokens(block: object) -> int:
"""
Token estimate for one ``input_audio`` content block.
When the block carries a base64 ``data`` payload, the estimate comes
from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``),
assuming the lowest reasonable audio bitrate so we never under-reserve
for higher-quality recordings of the same duration.
When no payload is present (reference-only block or missing ``data``),
falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``.
"""
if not isinstance(block, dict):
return DEFAULT_AUDIO_TOKEN_ESTIMATE
input_audio: Final = block.get("input_audio")
b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
if b64_data and isinstance(b64_data, str):
decoded_bytes: Final = len(b64_data) * 3 // 4
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
return DEFAULT_AUDIO_TOKEN_ESTIMATE
@classmethod
def _estimate_audio_content_tokens(cls, messages: object) -> int:
"""
Sum of per-block audio token estimates across all ``messages``.
Returns 0 when there are no ``input_audio`` blocks, which the caller
uses to skip the (relatively expensive) strip pass.
"""
if not isinstance(messages, list):
return 0
return sum(
cls._estimate_audio_block_tokens(block)
for message in messages
if isinstance(message, dict)
for content in (message.get("content"),)
if isinstance(content, list)
for block in content
if isinstance(block, dict) and block.get("type") == "input_audio"
def _estimate_image_block_tokens(
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
) -> int:
normalized: Final = as_openai_image_part(block)
image: Final = _callback_kwargs_mapping(normalized.get("image_url") if normalized is not None else None)
url: Final = image.get("url")
detail: Final = _callback_kwargs_mapping(block.get("image_url")).get("detail") or block.get("detail")
return estimate_image_tokens(
models,
url if isinstance(url, str) else "",
detail if isinstance(detail, str) else "auto",
fallback_model=fallback_model,
)
@staticmethod
def _strip_audio_content_blocks(messages: object) -> object:
"""
Drop ``input_audio`` content blocks before passing ``messages`` to
``token_counter``, which raises ``ValueError`` on them (no per-type
handling, unlike images). The audio contribution is added back
separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the
message (text/images/tools) still gets counted accurately instead of
the whole call falling back to the cheap char-count estimate.
"""
if not isinstance(messages, list):
return messages
sanitized: Final[list[object]] = [] # mutable-ok: token_counter requires a list of message dicts
for message in messages:
if not isinstance(message, dict):
sanitized.append(message)
continue
content = message.get("content")
if not isinstance(content, list):
sanitized.append(message)
continue
filtered_content = [
block for block in content if not (isinstance(block, dict) and block.get("type") == "input_audio")
]
sanitized.append({**message, "content": filtered_content})
return sanitized
def _estimate_audio_block_tokens(
block: object, models: tuple[str, ...] = (), fallback_model: str | None = None
) -> int:
audio: Final = _callback_kwargs_mapping(_callback_kwargs_mapping(block).get("input_audio"))
data: Final = audio.get("data")
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
@staticmethod
def _requires_input_count_offload(data: Mapping[str, object], call_type: str | None) -> bool:
if call_type in RESPONSES_API_CALL_TYPES + GOOGLE_GENAI_NATIVE_CALL_TYPES:
return True
messages: Final = data.get("messages")
return isinstance(messages, list) and any(
isinstance(_callback_kwargs_mapping(message).get("content"), list)
for message in _TOKEN_COUNT_ITEMS.validate_python(messages)
)
@staticmethod
def _responses_input_to_chat_messages(data: object) -> Sequence[object]:
@ -3873,93 +3837,110 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
state: Final = data.get("state")
return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}"
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
"""
Model-aware input token estimate for the project ITPM reservation,
using ``litellm.token_counter`` -- the same approach the
deployment-level itpm/otpm check uses in
``io_token_rate_limit_check.py``. Unlike the cheap char-count
estimate the combined-TPM path uses, this accounts for image/tool
content and derives per-``input_audio``-block estimates from the
base64 payload size (assuming the lowest reasonable bitrate so
longer recordings always reserve proportionally more), so a burst
of multimodal, tool-heavy, or audio-heavy requests can't each
reserve only the one-token floor and blow past ITPM before
post-call reconciliation catches up.
For the Responses API, ``input`` is converted to chat messages first
(via ``_responses_input_to_chat_messages``) so its own multimodal
content blocks are counted the same way; ``token_counter``'s ``text``
argument can only see plain strings in a list, not content blocks.
Falls back to the cheap char-count estimate if ``token_counter``
can't resolve a tokenizer for this model (e.g. an unrecognized
custom model name) or otherwise raises -- the audio add-on still
applies on top of the fallback.
"""
def _estimate_request_input_tokens(
self,
data: object,
model: str | None = None,
call_type: str | None = None,
models: tuple[str, ...] = (),
precise: bool = False,
) -> RequestInputTokenEstimate:
from litellm import token_counter
if not isinstance(data, dict):
return 0
if not isinstance(data, Mapping):
return RequestInputTokenEstimate(0, 0 if precise else None)
request: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(data)
is_responses_request: Final = call_type in RESPONSES_API_CALL_TYPES
translated_request: Final = (
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
translated: Final = (
None if is_responses_request else self._translate_google_genai_native_request(request, call_type)
)
is_embedding_request: Final = self._is_embedding_request(data, call_type)
embedding_text: Final = data.get("input") if is_embedding_request else None
pretokenized_input_tokens: Final = (
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
)
if pretokenized_input_tokens is not None:
return pretokenized_input_tokens
embedding_input: Final = request.get("input") if self._is_embedding_request(request, call_type) else None
pretokenized_tokens: Final = self._count_pretokenized_embedding_input(embedding_input)
is_rerank_request: Final = call_type in RERANK_API_CALL_TYPES
prompt: Final = data.get("prompt")
fallback_text: Final = prompt if prompt is not None else data.get("input")
selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = (
(self._responses_input_to_chat_messages(data), None, data.get("tools"), data.get("tool_choice"))
prompt: Final = request.get("prompt")
selected: Final[tuple[object, object, object, object]] = (
(self._responses_input_to_chat_messages(request), None, request.get("tools"), request.get("tool_choice"))
if is_responses_request
else (
translated_request.get("messages"),
None,
translated_request.get("tools"),
translated_request.get("tool_choice"),
)
if translated_request is not None
else (None, embedding_text, data.get("tools"), data.get("tool_choice"))
if is_embedding_request
else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice"))
if call_type in RERANK_API_CALL_TYPES
else (None, prompt, data.get("tools"), data.get("tool_choice"))
else (translated.get("messages"), None, translated.get("tools"), translated.get("tool_choice"))
if translated is not None
else (None, embedding_input, request.get("tools"), request.get("tool_choice"))
if self._is_embedding_request(request, call_type)
else (None, self._rerank_input_to_text(request), request.get("tools"), request.get("tool_choice"))
if is_rerank_request
else (None, prompt, request.get("tools"), request.get("tool_choice"))
if call_type in TEXT_COMPLETION_API_CALL_TYPES
else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(data)), None, None)
else (None, self._extraction_input_text(request), None, None)
if call_type in EXTRACTION_API_CALL_TYPES
else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice"))
)
messages, selected_text, countable_tools, countable_tool_choice = selected_inputs
audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
try:
estimate: Final = max(
0,
int(
token_counter(
model=model or "",
messages=countable_messages,
text=selected_text,
tools=countable_tools,
tool_choice=countable_tool_choice,
use_default_image_token_count=True,
)
),
else (
request.get("messages"),
None if call_type else prompt if prompt is not None else request.get("input"),
request.get("tools"),
request.get("tool_choice"),
)
return estimate + audio_token_estimate
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
return estimated_input_tokens + audio_token_estimate
)
messages, text, tools, tool_choice = selected
countable_messages, media = split_message_media(messages)
model_value: Final = request.get("model")
fallback_model: Final = model if model is not None else model_value if isinstance(model_value, str) else None
media_tokens: Final = sum(
self._estimate_audio_block_tokens(block, models, fallback_model)
if block.get("type") == "input_audio"
else self._estimate_image_block_tokens(block, models, fallback_model)
for block in media
)
message_text, opaque_document_sizes = get_token_count_text_and_document_sizes(
countable_messages, include_tool_arguments=is_responses_request
)
text_chars: Final = (
len(message_text)
if isinstance(messages, list) and messages
else len(text)
if isinstance(text, str)
else sum(len(str(item)) for item in _TOKEN_COUNT_ITEMS.validate_python(text))
if isinstance(text, list)
else 0
)
cheap_chars: Final = (
len(get_token_count_text_from_messages(request.get("messages"))) if is_rerank_request else text_chars
)
# Keep opaque bytes' character fallback until precise counting can read their pages.
cheap_tokens: Final = (
(max(1, cheap_chars // DEFAULT_CHARS_PER_TOKEN) if cheap_chars > 0 else 0)
+ media_tokens
+ sum(
max(DEFAULT_IMAGE_TOKEN_COUNT, (size + DEFAULT_CHARS_PER_TOKEN - 1) // DEFAULT_CHARS_PER_TOKEN)
for size in opaque_document_sizes
)
)
if not precise:
return RequestInputTokenEstimate(cheap_tokens)
if pretokenized_tokens is not None:
return RequestInputTokenEstimate(cheap_tokens, pretokenized_tokens)
try:
precise_tokens: Final = (
max(
0,
int(
token_counter(
model=fallback_model or "",
messages=_TOKEN_COUNT_SEQUENCE.validate_python(countable_messages),
text=_TOKEN_COUNT_TEXT.validate_python(text),
tools=_TOKEN_COUNT_SEQUENCE.validate_python(tools),
tool_choice=tool_choice,
use_default_image_token_count=True,
)
),
)
+ media_tokens
)
except Exception: # noqa: BLE001 # tokenizer failures use the already measured input
fallback_tokens: Final = text_chars // DEFAULT_CHARS_PER_TOKEN if is_rerank_request else cheap_tokens
return RequestInputTokenEstimate(cheap_tokens, fallback_tokens)
return RequestInputTokenEstimate(cheap_tokens, precise_tokens)
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
return self._estimate_request_input_tokens(data, model, call_type, precise=True).precise_tokens or 0
async def _reserve_project_io_tokens_or_raise(
self,
@ -3972,6 +3953,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type: str | None = None,
endpoint_type: EndpointType = EndpointType.GENERIC,
reuses_request_slot: bool = False,
input_estimate: RequestInputTokenEstimate | None = None,
) -> None:
"""
Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style
@ -3999,15 +3981,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if v is not None
]
min_configured_otpm_limit: Final = min(configured_otpm_limits) if configured_otpm_limits else None
measured_input: Final = (
input_estimate
if input_estimate is not None
else await offload_token_count(self._estimate_request_input_tokens)(
data=_REQUEST_RATE_LIMIT_DATA.validate_python(data),
model=requested_model,
call_type=call_type,
models=self._media_model_resolver(requested_model or "", user_api_key_dict.team_id),
precise=any(d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY for d in io_token_descriptors),
)
)
_, raw_estimated_output_tokens = self._estimate_input_and_output_tokens(
data=data,
min_configured_tpm_limit=min_configured_otpm_limit,
call_type=call_type,
input_estimate=measured_input,
)
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=requested_model, call_type=call_type
estimated_input_tokens: Final = max(
measured_input.precise_tokens if measured_input.precise_tokens is not None else measured_input.cheap_tokens,
1,
)
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
estimated_output_tokens: Final = (
raw_estimated_output_tokens
if self._has_explicit_output_cap(data, call_type)
@ -4323,6 +4317,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
has_tpm_limits: Final = bool(configured_tpm_limits)
needs_precise_input: Final = any(d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY for d in descriptors)
needs_input_estimate: Final = (has_tpm_limits and self.tpm_reservation_enabled) or any(
d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for d in descriptors
)
offload_input: Final = self._requires_input_count_offload(request_data, call_type)
input_estimate: Final = (
await offload_token_count(self._estimate_request_input_tokens)(
data=request_data,
model=requested_model,
call_type=call_type,
models=self._media_model_resolver(requested_model or "", user_api_key_dict.team_id)
if offload_input
else (),
precise=needs_precise_input,
)
if needs_input_estimate and (offload_input or needs_precise_input)
else self._estimate_request_input_tokens(request_data, requested_model, call_type)
if needs_input_estimate
else None
)
# Populated on a successful combined-TPM reservation below, so the
# project ITPM/OTPM block further down can roll it back if a
# different bucket in the same request subsequently hits its
@ -4366,6 +4381,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
input_estimate=input_estimate,
),
1,
)
@ -4390,6 +4406,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type=call_type,
configured_output_tokens=configured_output_tokens,
raw_estimate=estimated_tokens,
input_estimate=input_estimate,
)
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
if not reuses_request_slot:
@ -4453,6 +4470,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
user_api_key_dict=user_api_key_dict,
tpm_reservation_scopes=tpm_reservation_scopes,
tpm_reservation_amount=tpm_reservation_amount,
input_estimate=input_estimate,
call_type=call_type,
endpoint_type=endpoint_type,
reuses_request_slot=reuses_request_slot,

View file

@ -10224,28 +10224,38 @@ class Router:
model_info[field] = backend_value
@staticmethod
def _cost_map_backend_model(deployment: Deployment) -> str:
model_info_base_model: Final = deployment.model_info.base_model
def _cost_map_backend_model(deployment: Deployment | DeploymentTypedDict) -> str:
model_info_base_model: Final = (
deployment.model_info.base_model
if isinstance(deployment, Deployment)
else _configured_model_info(deployment).get("base_model")
)
if isinstance(model_info_base_model, str) and model_info_base_model:
return model_info_base_model
params_base_model: Final = deployment.litellm_params.get("base_model")
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(
deployment.litellm_params.model_dump(exclude_none=True)
if isinstance(deployment, Deployment)
else deployment["litellm_params"]
)
params_base_model: Final = params.get("base_model")
if isinstance(params_base_model, str) and params_base_model:
return params_base_model
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True))
model: Final = (
deployment.litellm_params.model
if isinstance(deployment, Deployment)
else deployment["litellm_params"].get("model", "")
)
invoked_model: Final = params.get("model_id")
if (
isinstance(invoked_model, str)
and invoked_model
and (
deployment.litellm_params.custom_llm_provider == "bedrock"
or deployment.litellm_params.model.startswith("bedrock/")
)
and (params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/"))
):
profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}"
profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock")
if not isinstance(profile_info, ModelInfoLookupMiss):
return profile_model
return deployment.litellm_params.model
return model
@staticmethod
def _inherit_builtin_service_tier_pricing(
@ -13795,7 +13805,9 @@ class Router:
]
return named or self._get_wildcard_deployments(model_name=model_group)
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
def resolved_litellm_models(
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
) -> tuple[str, ...]:
"""The provider model strings `model_name` can actually be served by on this proxy.
`get_model_list` composes every channel the request path itself uses (exact name,
@ -13808,6 +13820,12 @@ class Router:
deployment behind it - so the fallback for an empty result is the caller's policy,
never this function's.
"""
if use_base_model:
return tuple(
backend_model
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
if (backend_model := self._cost_map_backend_model(deployment))
)
return tuple(
litellm_model
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()

View file

@ -224,7 +224,6 @@ from litellm.types.llms.openai import (
AllMessageValues,
AllPromptValues,
ChatCompletionAssistantToolCall,
ChatCompletionNamedToolChoiceParam,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
OpenAITextCompletionUserMessage,
@ -2822,10 +2821,10 @@ def token_counter(
model="",
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
text: str | list[str] | None = None,
messages: Sequence | None = None,
messages: Sequence[object] | None = None,
count_response_tokens: bool | None = False,
tools: list[ChatCompletionToolParam] | None = None,
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
tools: Sequence[object] | None = None,
tool_choice: object = None,
use_default_image_token_count: bool | None = False,
default_token_count: int | None = None,
) -> int:

View file

@ -22,6 +22,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_format_from_file_id,
get_semantic_cache_prompt_from_messages,
get_str_from_messages,
get_token_count_text_and_document_sizes,
get_token_count_text_from_messages,
handle_any_messages_to_chat_completion_str_messages_conversion,
hoist_images_from_tool_messages,
is_encrypted_reasoning_block,
@ -35,6 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
schema_has_unresolvable_ref,
schema_property_count,
split_concatenated_json_objects,
split_message_media,
strip_encrypted_reasoning_from_messages,
strip_litellm_response_only_fields,
system_messages_first,
@ -2465,6 +2468,169 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch
assert schema_grammar_branch_count(schema) == branches
@pytest.mark.parametrize(
"media",
[
{"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}},
{"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"},
{"type": "image", "source": {"type": "file", "file_id": "opaque-image"}},
{"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}},
{"type": "image"},
{"type": "input_audio", "input_audio": None},
],
)
def test_split_message_media_preserves_every_nonmedia_sibling_and_media_detail(
media: Mapping[str, object],
) -> None:
siblings: Final = ["visible", None, 7, {"type": []}, {"type": "text", "text": "caption"}]
messages: Final = [{"role": "tool", "tool_call_id": "call_1", "content": [*siblings, media, media]}]
original: Final = copy.deepcopy(messages)
sanitized, leaves = split_message_media(messages)
assert sanitized == [{"role": "tool", "tool_call_id": "call_1", "content": siblings}]
assert leaves == (media, media)
assert messages == original
assert split_message_media(sanitized) == (sanitized, ())
def test_split_message_media_keeps_nested_tool_and_document_text_and_metadata() -> None:
direct: Final = {"type": "image_url", "image_url": "https://example.com/direct.png"}
image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}}
audio: Final = {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}
text: Final = {"type": "text", "text": "document text"}
file: Final = {"type": "file", "file": {"file_id": "opaque-file"}}
opaque: Final = {"type": "document", "source": {"type": "url", "url": "https://example.com/document.pdf"}}
document: Final = {
"type": "document", "title": "title", "context": "context", "citations": {"enabled": True},
"source": {"type": "content", "content": [text, image, file, "tail"]},
}
tool_result: Final = {
"type": "tool_result", "tool_use_id": "toolu_1", "is_error": False,
"content": ["tool text", audio, document, opaque],
}
messages: Final = [
{"role": "user", "content": [direct, tool_result]},
{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "toolu_2", "content": "result"},
{"type": "document", "source": {"type": "content", "content": "document"}},
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "declared"}},
]},
]
original: Final = copy.deepcopy(messages)
sanitized, leaves = split_message_media(messages)
assert sanitized == [{
"role": "user", "content": [{
**tool_result,
"content": ["tool text", {**document, "source": {"type": "content", "content": [text, file, "tail"]}}, opaque],
}],
}, messages[1]]
assert leaves == (direct, audio, image)
expected_text: Final = "tool texttitlecontextdocument texttailresultdocumentdeclared"
assert get_token_count_text_from_messages(sanitized) == expected_text
visible_text, opaque_sizes = get_token_count_text_and_document_sizes(sanitized)
assert visible_text == expected_text
assert len(opaque_sizes) == 2 and all(size > 0 for size in opaque_sizes)
assert messages == original
def test_split_message_media_does_not_walk_arbitrary_tool_arguments_or_invalid_nesting() -> None:
image: Final = {"type": "image", "source": {"type": "url", "url": "https://example.com/image.png"}}
document: Final = {"type": "document", "source": {"type": "content", "content": [image]}}
tool: Final = {"type": "tool_result", "content": [image]}
blocks: Final = [
{"type": "tool_use", "name": "inspect", "input": {"content": [image]}},
{"type": "unknown", "content": [image]},
{"type": "document", "source": {"type": "content", "content": [document]}},
{"type": "tool_result", "content": [tool]},
]
messages: Final = [{"role": "assistant", "content": blocks}]
sanitized, leaves = split_message_media(messages)
assert sanitized == messages
assert leaves == ()
def test_token_count_text_includes_only_opaque_assistant_argument_strings_when_requested() -> None:
arguments: Final = '{"type":"input_image","image_url":"data:image/png;base64,IMAGE"}'
calls: Final = [
{"function": {"arguments": arguments}},
{"function": {"arguments": {"text": "do not traverse"}}},
{"function": None},
None,
]
messages: Final = [
{"role": "assistant", "content": "before", "tool_calls": calls, "function_call": {"arguments": "legacy"}},
{"role": "user", "content": "after", "tool_calls": calls},
]
original: Final = copy.deepcopy(messages)
assert get_token_count_text_from_messages(messages) == "beforeafter"
assert get_token_count_text_from_messages(messages, include_tool_arguments=True) == (
"beforelegacy" + arguments + "after"
)
assert messages == original
@pytest.mark.parametrize(
("part", "expected"),
[
(
{"type": "file", "file": {"file_data": "data:image/png;base64,IMAGE", "detail": "low"}},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "low"}},
),
(
{"type": "input_file", "file_data": "data:image/png;base64,IMAGE", "detail": "low"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "low"}},
),
(
{"type": "file", "file": {"file_id": "opaque", "mime_type": "image/jpeg"}},
{"type": "image_url", "image_url": {"url": "opaque", "detail": "auto"}},
),
(
{"type": "file", "file": {"file_data": "data:audio/wav;base64,AUDIO"}},
{"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}},
),
(
{"type": "file", "file": {"file_data": "AUDIO", "content_type": "audio/mpeg"}},
{"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "mpeg"}},
),
(
{"type": "file", "file": {"file_id": "https://example.com/audio.wav", "format": "audio/wav"}},
{"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
),
],
)
def test_split_message_media_normalizes_only_explicit_image_and_audio_files(
part: Mapping[str, object], expected: Mapping[str, object],
) -> None:
messages: Final = [{"role": "user", "content": [part]}]
assert split_message_media(messages) == ([{"role": "user", "content": []}], (expected,))
assert get_token_count_text_and_document_sizes(messages) == ("", ())
@pytest.mark.parametrize(
"part",
[
{"type": "file", "file": {"file_data": "data:application/pdf;base64,PDF"}},
{"type": "file", "file": {"file_data": "data:video/mp4;base64,VIDEO"}},
{"type": "file", "file": {"file_id": "https://example.com/name.png"}},
{"type": "file", "file": {"file_data": "IMAGE", "format": "application/pdf"}},
{"type": "file", "file": None},
{"type": "thinking", "thinking": "thought", "signature": "opaque"},
{"type": "tool_use", "name": "read", "input": {"path": "file.txt"}},
{"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}},
],
)
def test_split_message_media_preserves_opaque_documents_and_unclassified_files(part: Mapping[str, object]) -> None:
messages: Final = [{"role": "user", "content": [part]}]
sanitized, leaves = split_message_media(messages)
assert sanitized == messages
assert leaves == ()
assert get_token_count_text_from_messages(messages) == ""
@pytest.mark.parametrize("part_type", ["image_url", "input_image"])
@pytest.mark.parametrize(
"fields",

View file

@ -2,16 +2,22 @@
Test the audio utils functionality in litellm_core_utils/audio_utils/utils.py
"""
import base64
import io
import os
import tempfile
from typing import Final
from unittest.mock import mock_open, patch
import pytest
import soundfile as sf
from litellm.litellm_core_utils.audio_utils.utils import (
AUDIO_BYTES_PER_TOKEN,
DEFAULT_AUDIO_TOKEN_ESTIMATE,
ProcessedAudioFile,
calculate_request_duration,
estimate_audio_tokens,
get_audio_file_content_hash,
get_audio_file_for_health_check,
get_audio_file_name,
@ -19,6 +25,74 @@ from litellm.litellm_core_utils.audio_utils.utils import (
)
def _encoded_audio(seconds: int, sample_rate: int, channels: int, container: str) -> str:
buffer: Final = io.BytesIO()
with sf.SoundFile(buffer, "w", samplerate=sample_rate, channels=channels, format=container) as audio:
audio.buffer_write(b"\0" * (seconds * sample_rate * channels * 2), dtype="int16")
return base64.b64encode(buffer.getvalue()).decode()
@pytest.mark.parametrize(
("sample_rate", "channels", "container"),
[
(8000, 1, "WAV"),
(48000, 2, "WAV"),
(24000, 1, "FLAC"),
(48000, 2, "OGG"),
(24000, 1, "MP3"),
],
)
def test_audio_estimate_uses_duration_across_containers_sample_rates_and_channels(
sample_rate: int, channels: int, container: str
) -> None:
short: Final = _encoded_audio(1, sample_rate, channels, container)
long: Final = _encoded_audio(31, sample_rate, channels, container)
assert estimate_audio_tokens(short) == 10
assert estimate_audio_tokens(long) == 310
# Audio rates verified 2026-10-26: https://ai.google.dev/gemini-api/docs/audio and
# https://developers.openai.com/api/docs/guides/voice-latency-cost?voice-api=realtime
@pytest.mark.parametrize(
("models", "fallback_model", "expected"),
[
(("gpt-audio-1.5",), None, 10),
(("gemini/gemini-3.8-flash",), None, 32),
(("vertex_ai/gemini-3.8-flash",), None, 32),
(("openrouter/google/gemini-3.8-flash",), None, 32),
(("gpt-audio-1.5", "gemini-3.8-flash"), None, 32),
(("azure/audio-deployment",), "gpt-audio-1.5", 10),
(("opaque-deployment",), "gemini-3.8-flash", 32),
(("openai/gpt-audio-1.5",), "gemini-3.8-flash", 10),
((), "gemini-3.8-flash", 32),
(("other/gemini-3.8-flash",), None, 10),
],
)
def test_audio_estimate_selects_the_backend_rate_before_the_requested_model(
models: tuple[str, ...], fallback_model: str | None, expected: int
) -> None:
assert estimate_audio_tokens(_encoded_audio(1, 8000, 1, "WAV"), models, fallback_model) == expected
@pytest.mark.parametrize("data", [None, "", "invalid-base64!", base64.b64encode(b"unreadable audio").decode()])
def test_audio_estimate_preserves_a_nonzero_fallback_for_unreadable_audio(data: str | None) -> None:
assert estimate_audio_tokens(data) == DEFAULT_AUDIO_TOKEN_ESTIMATE
def test_audio_estimate_preserves_size_based_fallback_for_large_unreadable_audio() -> None:
data: Final = base64.b64encode(b"\0" * (AUDIO_BYTES_PER_TOKEN * 400)).decode()
assert estimate_audio_tokens(data) == 400
@pytest.mark.parametrize("extra_bytes", [0, 1])
def test_audio_estimate_bounds_base64_allocation_before_decoding(extra_bytes: int) -> None:
data: Final = base64.b64encode(b"\0" * (16 * 1024 * 1024 + extra_bytes)).decode()
with patch("base64.b64decode", wraps=base64.b64decode) as decode_audio:
estimate: Final = estimate_audio_tokens(data)
assert estimate == len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN
assert decode_audio.call_count == (1 if extra_bytes == 0 else 0)
class TestProcessAudioFile:
"""Test the process_audio_file function with various input types"""

View file

@ -35,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import (
_get_extrapolating_count_function,
_get_tiktoken_count_function,
calculate_img_tokens,
estimate_image_tokens,
high_detail_image_token_upper_bound,
offload_token_count,
)
@ -2025,6 +2026,173 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_
assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound()
# Image policies verified 2026-10-24: https://developers.openai.com/api/docs/guides/images-vision
@pytest.mark.parametrize(
("model", "width", "height", "detail", "expected"),
[
("gpt-4o", 16, 16, "high", 255),
("gpt-4.1", 1024, 1024, "high", 765),
("gpt-4o-mini", 1024, 1024, "high", 25501),
("gpt-5.1", 1024, 1024, "auto", 630),
("gpt-5", 1024, 1024, "low", 70),
("o1", 1024, 1024, "high", 675),
("o1-pro", 1024, 1024, "high", 675),
("o3", 1024, 1024, "low", 75),
("openai/gpt-4o-2024-08-06", 2048, 4096, "auto", 1105),
("azure/gpt-4.1", 1024, 1024, "low", 85),
("openrouter/openai/gpt-4o-mini", 16, 16, "low", 2833),
("gpt-6-astra", 1024, 1024, "high", 1229),
("gpt-6-astra", 2048, 2048, "high", 3000),
("gpt-6-astra", 4096, 512, "high", 2458),
("gpt-6-astra", 2048, 2048, "auto", 4916),
("gpt-6-astra", 2048, 2048, "low", 308),
("gpt-5.6-sol", 4096, 512, "high", 615),
("gpt-5.6-terra", 2048, 2048, "original", 4916),
("gpt-5.6-luna", 2048, 2048, "auto", 4916),
("gpt-5.5", 6400, 6400, "original", 12000),
("gpt-5.4", 2048, 2048, "low", 4916),
("gpt-5.4-mini", 2048, 2048, "auto", 3000),
("gpt-5.4-nano", 6400, 6400, "original", 12000),
("gpt-5.2", 4096, 512, "auto", 615),
("gpt-4.1-mini-2025-04-14", 1024, 1024, "low", 1659),
("gpt-6-astra", 16, 16, "original", 2),
],
)
def test_image_estimate_uses_model_resolution_and_detail(
model: str, width: int, height: int, detail: str, expected: int
) -> None:
assert estimate_image_tokens(model, _png_data_url(width, height), detail) == expected
@pytest.mark.parametrize(
("model", "data", "detail"),
[
("unregistered-model", _png_data_url(1024, 1024), "high"),
("gpt-4.1-unrecognized", _png_data_url(1024, 1024), "high"),
("other/gpt-4o", _png_data_url(1024, 1024), "high"),
("gpt-4.1-nano", _png_data_url(1024, 1024), "high"),
("gpt-4o", _png_data_url(1024, 1024), "original"),
("gpt-5.2", _png_data_url(1024, 1024), "original"),
("gpt-6-astra", _png_data_url(1024, 1024), "invalid"),
],
)
def test_image_estimate_falls_back_when_policy_is_unavailable(model: str, data: str, detail: str) -> None:
assert estimate_image_tokens(model, data, detail) == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
# Unavailable dimensions use documented limits, verified 2026-10-26:
# https://developers.openai.com/api/docs/guides/images-vision (30,000 accepted patches before the multiplier)
@pytest.mark.parametrize(
("model", "detail", "expected", "data"),
[
("gpt-4o", "high", high_detail_image_token_upper_bound(), "https://image.invalid/photo.png"),
("gpt-4o-mini", "auto", high_detail_image_token_upper_bound(2833, 5667), "https://image.invalid/photo.png"),
("gpt-6-astra", "original", 36_000, "https://image.invalid/photo.png"),
("gpt-6-astra", "high", 3000, "https://image.invalid/photo.png"),
("gpt-6-astra", "low", 308, "https://image.invalid/photo.png"),
("gpt-4o-mini", "low", 2833, "https://image.invalid/photo.png"),
("gpt-6-astra", "high", 3000, "data:image/png;base64,invalid!"),
("gpt-6-astra", "high", 3000, "data:application/octet-stream;base64,bm90LWFuLWltYWdl"),
("gpt-6-astra", "high", 3000, _png_data_url(0, 1024)),
],
)
def test_image_estimate_uses_a_bounded_fallback_without_fetching_media(
model: str, detail: str, expected: int, data: str,
) -> None:
with patch("httpx.Client.send") as send:
estimate: Final = estimate_image_tokens(model, data, detail)
send.assert_not_called()
assert estimate == expected
assert estimate >= max(
estimate_image_tokens(model, _png_data_url(width, height), detail)
for width, height in ((16, 16), (1024, 1024), (4096, 4096), (768, 2000))
)
@pytest.mark.parametrize("mime_type", ["image/png", "application/octet-stream"])
def test_image_estimate_bounds_decoding_independent_of_payload_size(mime_type: str) -> None:
header: Final = base64.b64decode(_png_data_url(1024, 1024).split(",", 1)[1])
data: Final = f"data:{mime_type};base64," + base64.b64encode(header + b"\0" * 4_000_000).decode()
with patch("base64.b64decode", wraps=base64.b64decode) as decode_header:
estimate: Final = estimate_image_tokens(("gpt-6-astra", "azure/opaque"), data, "high", fallback_model="gpt-4o-mini")
assert estimate == estimate_image_tokens("gpt-4o-mini", _png_data_url(1024, 1024), "high")
assert decode_header.call_count == 1
assert len(decode_header.call_args.args[0]) <= 65_536
@pytest.mark.parametrize(
("model", "canonical"),
[
("openai/gpt-4o-mini-2026-10-01", "gpt-4o-mini"),
("azure/gpt-6-astra", "gpt-6-astra"),
("azure/openai/gpt-4o-mini", "gpt-4o-mini"),
("azure_ai/gpt-6-astra", "gpt-6-astra"),
("azure_ai/openai/gpt-4o-mini", "gpt-4o-mini"),
("openrouter/openai/gpt-6-astra", "gpt-6-astra"),
("azure/responses/gpt-4o-mini", "gpt-4o-mini"),
("azure/o_series/gpt-6-astra", "gpt-6-astra"),
("azure/gpt-41", "gpt-4.1"),
("azure/openai/gpt-41-mini", "gpt-4.1-mini"),
("other/openai/gpt-4o", "unknown"),
("azure/opaque/gpt-4o", "unknown"),
("openai/azure/gpt-4o", "unknown"),
("azure/openai/openai/gpt-4o", "unknown"),
("openrouter/other/gpt-4o", "unknown"),
("azure/gpt-4o-custom", "unknown"),
],
)
def test_image_estimate_resolves_only_supported_model_identities(model: str, canonical: str) -> None:
data: Final = _png_data_url(1024, 1024)
assert estimate_image_tokens(model, data, "high") == estimate_image_tokens(canonical, data, "high")
def test_image_estimate_recognizes_cataloged_azure_names_without_caching_unknowns() -> None:
model: Final = "azure/image-policy-test-region/gpt-4o-mini"
data: Final = _png_data_url(1024, 1024)
assert estimate_image_tokens(model, data, "high") == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
litellm.register_model(
{model: {"litellm_provider": "azure", "mode": "chat", "input_cost_per_token": 0, "output_cost_per_token": 0}}
)
assert estimate_image_tokens(model, data, "high") == estimate_image_tokens("gpt-4o-mini", data, "high")
@pytest.mark.parametrize(
("models", "fallback", "expected_model"),
[
(("azure/deployment-prod",), "gpt-4o", "gpt-4o"),
(("openai/gpt-4o",), "gpt-4o-mini", "gpt-4o"),
(("azure/deployment-prod", "openai/gpt-4o"), "gpt-4o-mini", "gpt-4o-mini"),
(("openai/gpt-4o", "openai/gpt-4o-mini"), None, "gpt-4o-mini"),
((), "gpt-6-astra", "gpt-6-astra"),
(("other/gpt-4o-mini",), "unregistered-model", "unregistered-model"),
(("azure/openai/gpt-4o",), "gpt-4o-mini", "gpt-4o"),
(("azure/openai/deployment-prod",), "azure_ai/openai/gpt-4o-mini", "gpt-4o-mini"),
(("azure/openai/gpt-4o", "azure/openai/deployment-prod"), "gpt-4o-mini", "gpt-4o-mini"),
],
)
def test_image_estimate_prefers_each_backend_policy_over_the_requested_model(
models: tuple[str, ...], fallback: str | None, expected_model: str
) -> None:
data: Final = _png_data_url(1024, 1024)
assert estimate_image_tokens(models, data, "high", fallback_model=fallback) == estimate_image_tokens(
expected_model, data, "high"
)
def test_image_estimate_falls_back_when_dimensions_are_beyond_header_budget() -> None:
jpeg: Final = (
b"\xff\xd8\xff\xe1\xff\xff"
+ b"\0" * 65533
+ b"\xff\xc0\x00\x0b\x08"
+ (512).to_bytes(2, "big")
+ (1024).to_bytes(2, "big")
)
data: Final = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
assert estimate_image_tokens("gpt-4o", data, "high") == high_detail_image_token_upper_bound()
HUB_SETUP_SCRIPT: Final = """
import json
import sys

View file

@ -15,19 +15,29 @@ Redis.
"""
import asyncio
import base64
import io
import wave
from functools import partial
import json
from datetime import datetime, timedelta
from typing import Any, Dict, Final
from typing import Any, Dict, Final, Literal
import pytest
from fastapi import HTTPException
from pydantic import TypeAdapter
from litellm import Router, token_counter
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
from litellm.litellm_core_utils.audio_utils.utils import AUDIO_BYTES_PER_TOKEN
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROJECT_ITPM_DESCRIPTOR_KEY,
PROJECT_OTPM_DESCRIPTOR_KEY,
RateLimitedModel,
_AUDIO_BYTES_PER_TOKEN,
_resolve_media_models,
PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler,
)
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
@ -37,6 +47,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
get_request_stash,
)
from litellm.proxy.utils import InternalUsageCache, hash_token
from litellm.router_utils.ptu_shares import PTUTeamCeiling
from litellm.types.llms.openai import (
InputTokensDetails,
ResponseAPIUsage,
@ -53,6 +64,397 @@ def rate_limiter():
return handler, cache
@pytest.mark.parametrize("output_cap,configured_output", [(None, None), (None, 32), (0, 32), (16, None)])
def test_media_only_input_keeps_the_existing_output_budget(
rate_limiter: tuple[RateLimitHandler, DualCache], output_cap: int | None, configured_output: int | None,
) -> None:
handler, _ = rate_limiter
data: Final = {
"messages": [{"role": "user", "content": [{"type": "image", "source": {"type": "file", "file_id": "opaque"}}]}],
"n": 2, **({"max_tokens": output_cap} if output_cap is not None else {}),
}
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
data, call_type="acompletion", configured_output_tokens=configured_output,
)
expected_output: Final = output_cap if output_cap is not None else configured_output or handler.no_max_tokens_output_floor(None)
assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT
assert output_tokens == expected_output * 2
def _audio_message(seconds: int, sample_rate: int, channels: int) -> dict[str, object]:
output: Final = io.BytesIO()
with wave.open(output, "wb") as wav:
wav.setnchannels(channels)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes(b"\x00\x00" * seconds * sample_rate * channels)
return {"role": "user", "content": [{
"type": "input_audio", "input_audio": {
"data": base64.b64encode(output.getvalue()).decode(), "format": "wav",
},
}]}
@pytest.mark.asyncio
@pytest.mark.parametrize("detail,admitted", [("low", True), ("high", False)])
async def test_model_specific_images_resolve_routes_before_admission(
detail: str, admitted: bool, equivalent_image_urls: tuple[str, str],
) -> None:
router: Final = Router(
model_list=[{
"model_name": "vision", "litellm_params": {
"model": "azure/custom-deployment", "base_model": "gpt-4o-mini",
"api_key": "test", "api_base": "https://example.invalid", "api_version": "2024-12-01-preview",
}, "model_info": {"base_model": "gpt-6-astra"},
}], model_group_alias={"alias": "vision"},
)
cache: Final = DualCache()
handler: Final = RateLimitHandler(
internal_usage_cache=InternalUsageCache(cache), media_model_resolver=partial(_resolve_media_models, router),
)
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-image-policy"), tpm_limit=500)
data: Final = {"model": "alias", "messages": [{"role": "user", "content": [
{"type": "image_url", "image_url": {"url": equivalent_image_urls[1], "detail": detail}},
]}], "max_tokens": 16}
if not admitted:
with pytest.raises(ProxyRateLimitError):
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
return
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
stash: Final = get_request_stash()
assert stash is not None
assert 250 < stash.reserved_tokens < 500
@pytest.fixture(scope="module")
def equivalent_image_urls() -> tuple[str, str]:
from PIL import Image
def encoded_image(compression: int) -> str:
output: Final = io.BytesIO()
Image.new("RGB", (1024, 1024)).save(output, format="PNG", compress_level=compression)
return "data:image/png;base64," + base64.b64encode(output.getvalue()).decode()
return encoded_image(0), encoded_image(9)
def _structured_image_request(
image_url: str, representation: str, detail: str = "high"
) -> tuple[str, dict[str, object]]:
image: Final = {"type": "image_url", "image_url": {"url": image_url, "detail": detail}}
text: Final = {"type": "text", "text": "Describe this image"}
base: Final = {"model": "gpt-6-astra"}
if representation == "chat":
return "acompletion", {**base, "messages": [{"role": "user", "content": [text, image]}], "max_tokens": 16}
if representation == "chat_input_image":
input_image: Final = {**image, "type": "input_image"}
return "acompletion", {**base, "messages": [{"role": "user", "content": [text, input_image]}], "max_tokens": 16}
if representation == "anthropic_tool":
source_image: Final = {"type": "image", "source": {
"type": "base64", "media_type": "image/png", "data": image_url.partition(",")[2],
}}
return "anthropic_messages", {**base, "messages": [{"role": "user", "content": [{
"type": "tool_result", "tool_use_id": "call_image", "content": [text, source_image],
}]}], "max_tokens": 16}
if representation == "document":
return "anthropic_messages", {**base, "messages": [{"role": "user", "content": [{
"type": "document", "source": {"type": "content", "content": [text, image]},
}]}], "max_tokens": 16}
response_content: Final = [
{"type": "input_text", "text": "Describe this image"},
{"type": "input_image", "image_url": image_url, "detail": detail},
]
response_input: Final = (
[{"type": "custom_tool_call_output", "call_id": "call_image", "output": response_content}]
if representation == "custom_output"
else [{"role": "user", "content": response_content}]
)
return "aresponses", {**base, "input": response_input, "max_output_tokens": 16}
@pytest.mark.parametrize("representation", ["chat", "chat_input_image", "anthropic_tool", "document", "responses", "custom_output"])
def test_structured_media_estimates_ignore_transport_size_in_every_quota(
rate_limiter: tuple[RateLimitHandler, DualCache], equivalent_image_urls: tuple[str, str], representation: str,
) -> None:
handler, _ = rate_limiter
uncompressed, compressed = equivalent_image_urls
assert len(uncompressed) > 100 * len(compressed)
call_type, first = _structured_image_request(uncompressed, representation)
_, second = _structured_image_request(compressed, representation)
original: Final = json.dumps(first)
first_input: Final = handler._estimate_request_input_tokens(first, call_type=call_type, precise=True)
second_input: Final = handler._estimate_request_input_tokens(second, call_type=call_type, precise=True)
assert first_input == second_input
assert 500 < first_input.cheap_tokens < 5000
assert first_input.precise_tokens is not None and 500 < first_input.precise_tokens < 5000
assert handler._estimate_input_and_output_tokens(first, call_type=call_type) == (
first_input.cheap_tokens, 16,
)
assert json.dumps(first) == original
@pytest.mark.parametrize("tool_kind", ["function_call", "custom_tool_call"])
def test_responses_tool_arguments_grow_reservations(
rate_limiter: tuple[RateLimitHandler, DualCache],
equivalent_image_urls: tuple[str, str],
tool_kind: str,
) -> None:
handler, _ = rate_limiter
output_request: Final = _structured_image_request(equivalent_image_urls[1], "custom_output")[1]
tool_output: Final = TypeAdapter(list[dict[str, object]]).validate_python(output_request["input"])[0]
requests: Final = tuple(
{
**output_request,
"input": [
{
"type": tool_kind, "call_id": "call_image", "name": "inspect",
**(
{"arguments": json.dumps({"query": "a" * size})}
if tool_kind == "function_call" else {"input": "a" * size}
),
},
{
**tool_output,
"type": tool_kind + "_output",
},
],
}
for size in (8, 8008)
)
estimates: Final = tuple(
handler._estimate_request_input_tokens(data, call_type="aresponses", precise=True) for data in requests
)
assert estimates[1].cheap_tokens - estimates[0].cheap_tokens == 2000
chat_estimates: Final = tuple(
handler._estimate_request_input_tokens(
{"model": "gpt-6-astra", "messages": handler._responses_input_to_chat_messages(data)},
call_type="acompletion",
).cheap_tokens
for data in requests
)
assert chat_estimates[0] == chat_estimates[1]
with_tools: Final = {
**requests[1], "tools": [{"type": "function", "name": "inspect", "description": "d" * 12000}]
}
assert (
handler._estimate_request_input_tokens(with_tools, call_type="aresponses").cheap_tokens
== estimates[1].cheap_tokens
)
unsupported_content: Final = {
**requests[1], "input": [
*TypeAdapter(list[dict[str, object]]).validate_python(requests[1]["input"]),
{"role": "user", "content": [
{"type": "guarded_text", "text": ""},
{"type": "input_file", "filename": "opaque.pdf", "file_data": "data:application/pdf;base64,eA=="},
]},
],
}
with pytest.raises(ValueError, match="guarded_text"):
token_counter(
model="gpt-6-astra", messages=handler._responses_input_to_chat_messages(unsupported_content),
use_default_image_token_count=True,
)
fallback: Final = handler._estimate_request_input_tokens(unsupported_content, call_type="aresponses", precise=True)
assert fallback.precise_tokens == fallback.cheap_tokens == estimates[1].cheap_tokens + DEFAULT_IMAGE_TOKEN_COUNT
@pytest.mark.parametrize("representation", ["chat", "responses", "anthropic_tool", "document_content"])
def test_opaque_documents_keep_a_cheap_reservation_and_existing_precise_count(
rate_limiter: tuple[RateLimitHandler, DualCache], representation: str,
) -> None:
from pypdf import PdfWriter
handler, _ = rate_limiter
writer: Final = PdfWriter()
writer.add_blank_page(width=612, height=792)
writer.add_blank_page(width=612, height=792)
output: Final = io.BytesIO()
writer.write(output)
encoded: Final = base64.b64encode(output.getvalue()).decode()
file: Final = {"type": "file", "file": {"filename": "document.pdf", "file_data": "data:application/pdf;base64," + encoded}}
document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": encoded}}
content: Final = (
[{"type": "tool_result", "tool_use_id": "read", "content": [document, document]}]
if representation == "anthropic_tool"
else [{"type": "document", "source": {"type": "content", "content": [file, file]}}]
if representation == "document_content"
else [file, file]
)
request: Final = (
{"model": "gpt-6-astra", "input": [{"role": "user", "content": [
{"type": "input_file", **file["file"]}, {"type": "input_file", **file["file"]},
]}], "max_output_tokens": 16}
if representation == "responses"
else {"model": "gpt-6-astra", "messages": [{"role": "user", "content": content}], "max_tokens": 16}
)
call_type: Final = "aresponses" if representation == "responses" else "acompletion"
original: Final = json.dumps(request)
estimate: Final = handler._estimate_request_input_tokens(request, call_type=call_type, precise=True)
assert estimate.cheap_tokens == 2 * DEFAULT_IMAGE_TOKEN_COUNT
assert handler._estimate_input_and_output_tokens(request, call_type=call_type) == (estimate.cheap_tokens, 16)
messages: Final = handler._responses_input_to_chat_messages(request) if representation == "responses" else request["messages"]
assert estimate.precise_tokens == token_counter(
model="gpt-6-astra", messages=messages, use_default_image_token_count=True,
)
assert estimate.precise_tokens > estimate.cheap_tokens
assert json.dumps(request) == original
for _ in range(18):
writer.add_blank_page(width=612, height=792)
large_output: Final = io.BytesIO()
writer.write(large_output)
large_encoded: Final = base64.b64encode(large_output.getvalue()).decode()
large_request: Final = TypeAdapter(dict[str, object]).validate_json(original.replace(encoded, large_encoded))
large_estimate: Final = handler._estimate_request_input_tokens(large_request, call_type=call_type)
assert large_estimate.cheap_tokens > estimate.cheap_tokens
assert large_estimate.cheap_tokens >= len(large_encoded) // 2
@pytest.mark.asyncio
@pytest.mark.parametrize("quota", ["combined", "project", "ptu"])
@pytest.mark.parametrize("detail,limit,admitted", [("low", 500, True), ("high", 500, False), ("high", 2000, True)])
async def test_image_detail_controls_each_admission_path(
equivalent_image_urls: tuple[str, str], quota: str, detail: str, limit: int, admitted: bool,
) -> None:
def ceiling(_team: str, model: str) -> PTUTeamCeiling | None:
return PTUTeamCeiling(model_group=model, tpm_limit=limit, output_to_input_ratio=4.0, cached_input_ratio=0.0) if quota == "ptu" else None
cache: Final = DualCache()
handler: Final = RateLimitHandler(
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=ceiling,
)
auth: Final = UserAPIKeyAuth(
api_key=hash_token("sk-detail-quota"), team_id="media-team",
tpm_limit=limit if quota == "combined" else None, project_id="media-project",
project_metadata={"model_itpm_limit": {"gpt-6-astra": limit}} if quota == "project" else {},
)
call_type, data = _structured_image_request(equivalent_image_urls[1], "chat", detail)
if admitted:
await handler.async_pre_call_hook(auth, cache, data, call_type)
stash: Final = get_request_stash()
assert stash is not None
assert max(stash.reserved_tokens, stash.itpm_reserved_tokens, stash.ptu_reserved_tokens) > 250
return
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(auth, cache, data, call_type)
assert error.value.status_code == (400 if quota == "ptu" else 429)
def test_precise_tokenizer_fallback_reuses_nested_media_and_visible_text_once(
rate_limiter: tuple[RateLimitHandler, DualCache], equivalent_image_urls: tuple[str, str],
) -> None:
handler, _ = rate_limiter
call_type, request = _structured_image_request(equivalent_image_urls[1], "anthropic_tool")
data: Final = {**request, "messages": [
*TypeAdapter(list[object]).validate_python(request["messages"]),
_audio_message(31, 48000, 2),
{"role": "user", "content": [{"type": "unsupported_counter_block"}]},
]}
measured: Final = handler._estimate_request_input_tokens(data, call_type=call_type, precise=True)
image_only: Final = handler._estimate_request_input_tokens(request, call_type=call_type)
assert measured.precise_tokens == measured.cheap_tokens == image_only.cheap_tokens + 310
@pytest.mark.asyncio
@pytest.mark.parametrize("outcome", ["success", "failure", "partial", "project_rejection"])
async def test_mixed_media_reservations_settle_each_bucket_in_its_own_units(
equivalent_image_urls: tuple[str, str], outcome: Literal["success", "failure", "partial", "project_rejection"],
) -> None:
now: Final = datetime(2026, 1, 2, 3, 4, 5)
def ceiling(_team: str, model: str) -> PTUTeamCeiling:
return PTUTeamCeiling(model_group=model, tpm_limit=10000, output_to_input_ratio=4.0, cached_input_ratio=0.0)
cache: Final = DualCache()
handler: Final = RateLimitHandler(
internal_usage_cache=InternalUsageCache(cache), time_provider=lambda: now, ptu_team_ceiling_resolver=ceiling,
)
auth: Final = UserAPIKeyAuth(
api_key=hash_token("sk-media-settlement"), tpm_limit=10000, team_id="media-team", project_id="media-project",
project_metadata={
"model_itpm_limit": {"gpt-6-astra": 500 if outcome == "project_rejection" else 2000},
"model_otpm_limit": {"gpt-6-astra": 10000},
},
)
_, image_request = _structured_image_request(equivalent_image_urls[1], "chat")
data: Final = {**image_request, "messages": [*TypeAdapter(list[object]).validate_python(image_request["messages"]), _audio_message(31, 24000, 1)]}
scopes: Final = (
("api_key", auth.api_key),
("model_per_team_ptu", "media-team:gpt-6-astra"),
(PROJECT_ITPM_DESCRIPTOR_KEY, "media-project:gpt-6-astra"),
(PROJECT_OTPM_DESCRIPTOR_KEY, "media-project:gpt-6-astra"),
)
if outcome == "project_rejection":
with pytest.raises(ProxyRateLimitError):
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
else:
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
stash: Final = get_request_stash()
assert stash is not None
assert stash.reserved_tokens > 1500
assert stash.itpm_reserved_tokens > 1500
assert stash.ptu_reserved_tokens == stash.reserved_tokens + 3 * stash.otpm_reserved_tokens
usage: Final = Usage(
prompt_tokens=40, completion_tokens=10, total_tokens=50,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=15),
)
kwargs: Final = {
"litellm_call_id": data.get("litellm_call_id"),
"model": "gpt-6-astra",
"standard_logging_object": {"metadata": {
"user_api_key_hash": auth.api_key, "user_api_key_team_id": auth.team_id,
"user_api_key_project_id": auth.project_id,
}},
**({"combined_usage_object": usage} if outcome == "partial" else {}),
}
if outcome == "success":
response: Final = ModelResponse(model="gpt-6-astra", usage=usage, choices=[])
await handler.async_log_success_event(kwargs, response, now, now)
else:
await handler.async_log_failure_event(kwargs, None, now, now)
await handler.async_log_failure_event(kwargs, None, now, now)
expected: Final = (35, 65, 25, 10) if outcome in ("success", "partial") else (0, 0, 0, 0)
for (scope, value), amount in zip(scopes, expected):
key: Final = handler.create_rate_limit_keys(scope, value or "", "tokens")
assert int(await cache.async_get_cache(key=key, local_only=True) or 0) == amount
@pytest.mark.asyncio
@pytest.mark.parametrize("representation", ["chat", "inlineData"])
@pytest.mark.parametrize("seconds,admitted", [(1, True), (31, False)])
async def test_native_and_chat_audio_share_duration_based_quota_admission(
rate_limiter: tuple[RateLimitHandler, DualCache], representation: str, seconds: int, admitted: bool,
) -> None:
handler, cache = rate_limiter
model: Final = "gemini/gemini-3.8-flash"
message: Final = _audio_message(seconds, 24000, 1)
blocks: Final = TypeAdapter(list[dict[str, object]]).validate_python(message["content"])
audio: Final = TypeAdapter(dict[str, str]).validate_python(blocks[0]["input_audio"])
native: Final = representation != "chat"
data: Final = (
{"model": model, "contents": [{"role": "user", "parts": [{
"inlineData": {"mimeType": "audio/wav", "data": audio["data"]},
}]}], "config": {"maxOutputTokens": 16}}
if native
else {"model": model, "messages": [message], "max_tokens": 16}
)
auth: Final = UserAPIKeyAuth(
api_key=hash_token("sk-native-duration"), tpm_limit=100, project_id="audio-project",
project_metadata={"model_itpm_limit": {model: 100}},
)
call_type: Final = "agenerate_content" if native else "acompletion"
if admitted:
await handler.async_pre_call_hook(auth, cache, data, call_type)
stash: Final = get_request_stash()
assert stash is not None
assert 26 < stash.reserved_tokens < 100
assert 10 < stash.itpm_reserved_tokens < 100
return
with pytest.raises(ProxyRateLimitError):
await handler.async_pre_call_hook(auth, cache, data, call_type)
@pytest.fixture(autouse=True)
def _isolated_request_stash():
token = _request_stash.set(None)
@ -2274,7 +2676,7 @@ def test_audio_token_estimate_scales_with_payload_size():
to exhaust ITPM quota while reserving almost nothing.
The estimate must now grow proportionally with the base64 payload size
(len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN), floored at
(len(b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN), floored at
DEFAULT_AUDIO_TOKEN_ESTIMATE so reference-only blocks and genuinely
short clips still get a non-trivial reservation.
@ -2312,7 +2714,7 @@ def test_audio_token_estimate_scales_with_payload_size():
f"Large payload ({large_estimate}) must reserve more than small payload "
f"({small_estimate}); flat-rate bug is back"
)
assert very_large_estimate == len(very_large_b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN
assert very_large_estimate == len(very_large_b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN
assert very_large_estimate > 6_000
assert no_data_estimate >= 300, (
f"Reference-only block (no data) must use the DEFAULT_AUDIO_TOKEN_ESTIMATE floor; got {no_data_estimate}"
@ -2792,26 +3194,6 @@ async def test_reserve_io_tokens_itpm_only_no_otpm(rate_limiter):
assert otpm_reserved == 0
def test_strip_audio_content_blocks_passthrough_non_list_messages():
"""Non-list input is returned unchanged (early return on line 2605)."""
result = RateLimitHandler._strip_audio_content_blocks("not a list")
assert result == "not a list"
def test_strip_audio_content_blocks_passthrough_non_dict_message():
"""Non-dict entries in the message list are appended unchanged."""
messages = ["plain string message"]
result = RateLimitHandler._strip_audio_content_blocks(messages)
assert result == ["plain string message"]
def test_strip_audio_content_blocks_passthrough_non_list_content():
"""Messages with non-list content (e.g. plain string) pass through unchanged."""
messages = [{"role": "user", "content": "hello"}]
result = RateLimitHandler._strip_audio_content_blocks(messages)
assert result == [{"role": "user", "content": "hello"}]
@pytest.mark.asyncio
async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter):
"""
@ -3519,6 +3901,28 @@ def test_split_token_estimate_selects_endpoint_input(rate_limiter, call_type, da
assert estimated_output == 1
@pytest.mark.parametrize(
"call_type,data,cheap_input,precise_input",
[
("embedding", {"input": [1, 2, 3]}, 1, 3),
("embedding", {"input": [[1, 2], [3]]}, 2, 3),
("rerank", {"query": "query text", "documents": ["first document", "second document"]}, 0, None),
],
)
def test_nonmedia_input_keeps_distinct_cheap_and_precise_reservations(
rate_limiter: tuple[RateLimitHandler, DualCache], call_type: str, data: dict[str, object],
cheap_input: int, precise_input: int | None,
) -> None:
handler, _ = rate_limiter
estimate: Final = handler._estimate_request_input_tokens(data, "gpt-6-astra", call_type, precise=True)
assert handler._estimate_input_and_output_tokens(
data, min_configured_tpm_limit=100, call_type=call_type, input_estimate=estimate,
) == (cheap_input, 0)
assert estimate.precise_tokens is not None and estimate.precise_tokens > cheap_input
if precise_input is not None:
assert estimate.precise_tokens == precise_input
def test_split_quota_multimodal_guards_handle_non_mapping_inputs(rate_limiter):
handler, _cache = rate_limiter

View file

@ -20261,8 +20261,9 @@ def _resolution_router() -> Router:
],
ids=["exact-name", "model-group-alias", "every-member-of-a-pool", "wildcard-expands", "resolves-to-nothing"],
)
@pytest.mark.parametrize("use_base_model", (False, True))
def test_resolved_litellm_models_answers_through_every_channel_a_request_uses(
model_name: str, expected: tuple[str, ...]
model_name: str, expected: tuple[str, ...], use_base_model: bool
) -> None:
"""A caller comparing two names by what serves them needs each channel the request path
composes, since the deployment name an admin picked carries no information on its own.
@ -20270,7 +20271,7 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses(
`resolves-to-nothing` is the contract that keeps the fallback out of here: an empty
result is not "the call fails", so what to do about it stays each caller's policy.
"""
assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected)
assert set(_resolution_router().resolved_litellm_models(model_name, use_base_model=use_base_model)) == set(expected)
class TestTierParamsTheTargetAccepts:
@ -30149,3 +30150,67 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items():
assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True
assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False
assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False
@pytest.mark.parametrize("info_base,params_base", [(None, None), (None, "gpt-5.1"), ("gpt-6-astra", "gpt-5.1")])
def test_resolved_litellm_models_can_use_configured_base_model(
info_base: str | None, params_base: str | None,
) -> None:
router: Final = Router(
model_list=[{
"model_name": "deployment", "litellm_params": {
"model": "azure/custom-name", "api_key": "test", "api_base": "https://example.invalid",
"api_version": "2024-12-01-preview", **({"base_model": params_base} if params_base else {}),
}, "model_info": {"base_model": info_base}, "rpm": 10.5,
}], model_group_alias={"alias": "deployment"},
)
assert router.resolved_litellm_models("alias") == ("azure/custom-name",)
assert router.resolved_litellm_models("alias", use_base_model=True) == (
info_base or params_base or "azure/custom-name",
)
def test_resolved_model_identity_does_not_revalidate_stored_quotas() -> None:
router: Final = Router(
model_list=[{
"model_name": "vision/*",
"litellm_params": {
"model": "azure/vision-deployment", "api_key": "test", "api_base": "https://example.invalid",
"api_version": "2024-12-01-preview", "base_model": "gpt-5.1",
},
"model_info": {"base_model": "gpt-6-astra"},
"tpm": 10.5,
}],
ignore_invalid_deployments=False,
)
assert router.resolved_litellm_models("vision/image") == ("azure/vision-deployment",)
assert router.resolved_litellm_models("vision/image", use_base_model=True) == ("gpt-6-astra",)
@pytest.mark.parametrize("explicit_provider", (False, True))
@pytest.mark.parametrize("profile_known", (False, True))
def test_resolved_model_identity_uses_only_mapped_bedrock_profiles(
monkeypatch: pytest.MonkeyPatch, explicit_provider: bool, profile_known: bool,
) -> None:
profile: Final = "us.anthropic.identity-test"
model: Final = "converse/anthropic.identity-test"
monkeypatch.setattr(litellm, "model_cost", {
**litellm.model_cost,
**({profile: {
"litellm_provider": "bedrock", "mode": "chat",
"input_cost_per_token": 0.01, "output_cost_per_token": 0.02,
}} if profile_known else {}),
})
configured_model: Final = model if explicit_provider else f"bedrock/{model}"
router: Final = Router(model_list=[{
"model_name": "profile", "litellm_params": {
"model": configured_model, "model_id": profile,
**({"custom_llm_provider": "bedrock"} if explicit_provider else {}),
},
}])
assert router.resolved_litellm_models("profile") == (configured_model,)
assert router.resolved_litellm_models("profile", use_base_model=True) == (
f"bedrock/{profile}" if profile_known else configured_model,
)