mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge faed6b6bb6 into 477ea4ec7c
This commit is contained in:
commit
d5899eca09
11 changed files with 1565 additions and 254 deletions
|
|
@ -2,8 +2,12 @@
|
|||
Utils used for litellm.transcription() and litellm.atranscription()
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -331,6 +335,44 @@ def calculate_request_duration(file: FileTypes) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
|
||||
AUDIO_TOKENS_PER_SECOND: Final = 10
|
||||
AUDIO_BYTES_PER_TOKEN: Final = 1600
|
||||
_MAX_AUDIO_ESTIMATE_BYTES: Final = 16 * 1024 * 1024
|
||||
|
||||
|
||||
def _audio_token_rate(model: str) -> int | None:
|
||||
if model.startswith(
|
||||
("gemini-", "gemini/gemini-", "vertex_ai/gemini-", "vertex_ai_beta/gemini-", "openrouter/google/gemini-")
|
||||
):
|
||||
return 32
|
||||
if model.startswith(("gpt-", "openai/gpt-", "azure/gpt-", "openrouter/openai/gpt-")):
|
||||
return AUDIO_TOKENS_PER_SECOND
|
||||
return None
|
||||
|
||||
|
||||
def estimate_audio_tokens(data: str | None, models: Sequence[str] = (), fallback_model: str | None = None) -> int:
|
||||
"""Estimate audio tokens from container duration within a bounded decoding budget."""
|
||||
if not data:
|
||||
return DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
fallback_estimate: Final = max(len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
|
||||
padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
|
||||
if len(data) * 3 // 4 - padding > _MAX_AUDIO_ESTIMATE_BYTES:
|
||||
return fallback_estimate
|
||||
try:
|
||||
duration: Final = calculate_request_duration(base64.b64decode(data, validate=True))
|
||||
except (ValueError, binascii.Error):
|
||||
return fallback_estimate
|
||||
if duration is None or not math.isfinite(duration) or duration <= 0:
|
||||
return fallback_estimate
|
||||
fallback_rate: Final = _audio_token_rate(fallback_model) if fallback_model is not None else None
|
||||
rate: Final = max(
|
||||
(_audio_token_rate(model) or fallback_rate or AUDIO_TOKENS_PER_SECOND for model in models),
|
||||
default=fallback_rate or AUDIO_TOKENS_PER_SECOND,
|
||||
)
|
||||
return max(1, math.ceil(duration * rate))
|
||||
|
||||
|
||||
DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import json
|
|||
import mimetypes
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from graphlib import CycleError, TopologicalSorter
|
||||
from itertools import chain, groupby, islice
|
||||
from os import PathLike
|
||||
|
|
@ -234,6 +235,199 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
|
|||
_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
|
||||
_IMAGE_SCAN_MAX_DEPTH: Final = 4
|
||||
|
||||
_MEDIA_CONTENT_PART_TYPES: Final = _IMAGE_CONTENT_PART_TYPES | {"input_audio"}
|
||||
_REMOVED_MEDIA_PART: Final = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ContentMediaSplit:
|
||||
content: object
|
||||
media: tuple[Mapping[str, object], ...] = ()
|
||||
text: str = ""
|
||||
opaque_document_sizes: tuple[int, ...] = ()
|
||||
|
||||
|
||||
def _message_content_parts(value: object) -> Sequence[object] | None:
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return None
|
||||
return cast(Sequence[object], value) # cast-ok: finite JSON content arrays
|
||||
|
||||
|
||||
def _file_media_part(part: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
|
||||
file: Final = as_str_mapping(part.get("file")) if part.get("type") == "file" else part
|
||||
if file is None:
|
||||
return None
|
||||
data: Final = file.get("file_data")
|
||||
data_string: Final = data if isinstance(data, str) else ""
|
||||
declared_type: Final = next(
|
||||
(value for key in ("format", "mime_type", "content_type") if isinstance(value := file.get(key), str) and value),
|
||||
"",
|
||||
)
|
||||
mime_type: Final = (declared_type or parse_mime_type(data_string[:256]) or "").split(";", 1)[0].strip().lower()
|
||||
if mime_type.startswith("image/"):
|
||||
file_id: Final = file.get("file_id")
|
||||
url: Final = data_string or (file_id if isinstance(file_id, str) else "")
|
||||
image_url: Final = (
|
||||
url
|
||||
if not data_string or url.startswith(("data:", "https://", "http://", "gs://"))
|
||||
else f"data:{mime_type};base64,{url}"
|
||||
)
|
||||
return {"type": "image_url", "image_url": {"url": image_url, "detail": file.get("detail", "auto")}}
|
||||
if mime_type.startswith("audio/"):
|
||||
header_end: Final = data_string.find(",", 0, 256)
|
||||
encoded: Final = (
|
||||
data_string[header_end + 1 :]
|
||||
if header_end >= 0 and data_string[:header_end].endswith(";base64")
|
||||
else ""
|
||||
if data_string.startswith(("data:", "https://", "http://", "gs://"))
|
||||
else data_string
|
||||
)
|
||||
return {"type": "input_audio", "input_audio": {"data": encoded, "format": mime_type.removeprefix("audio/")}}
|
||||
return None
|
||||
|
||||
|
||||
def _opaque_document_size(payload: object) -> int:
|
||||
fields: Final = as_str_mapping(payload)
|
||||
strings: Final = {key: value for key, value in (fields or {}).items() if isinstance(value, str)}
|
||||
return len(str(strings))
|
||||
|
||||
|
||||
def _split_media_leaf(part: object) -> _ContentMediaSplit:
|
||||
if isinstance(part, str):
|
||||
return _ContentMediaSplit(part, text=part)
|
||||
block: Final = as_str_mapping(part)
|
||||
if block is None:
|
||||
return _ContentMediaSplit(part)
|
||||
part_type: Final = block.get("type")
|
||||
media: Final = (
|
||||
block
|
||||
if isinstance(part_type, str) and part_type in _MEDIA_CONTENT_PART_TYPES
|
||||
else _file_media_part(block)
|
||||
if part_type in ("file", "input_file")
|
||||
else None
|
||||
)
|
||||
if media is not None:
|
||||
return _ContentMediaSplit(_REMOVED_MEDIA_PART, (media,))
|
||||
text: Final = block.get("text")
|
||||
return _ContentMediaSplit(
|
||||
part,
|
||||
text=text if isinstance(text, str) else "",
|
||||
opaque_document_sizes=(
|
||||
(_opaque_document_size(block.get("file") if part_type == "file" else block),)
|
||||
if part_type in ("file", "input_file")
|
||||
else ()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _split_media_content(content: object, split_part: Callable[[object], _ContentMediaSplit]) -> _ContentMediaSplit:
|
||||
parts: Final = _message_content_parts(content)
|
||||
if parts is None:
|
||||
return _ContentMediaSplit(content, text=content if isinstance(content, str) else "")
|
||||
splits: Final = tuple(map(split_part, parts))
|
||||
media: Final = tuple(chain.from_iterable(split.media for split in splits))
|
||||
kept: Final = tuple(split.content for split in splits if split.content is not _REMOVED_MEDIA_PART)
|
||||
sanitized: Final = content if not media else kept if isinstance(content, tuple) else list(kept)
|
||||
return _ContentMediaSplit(
|
||||
sanitized,
|
||||
media,
|
||||
"".join(split.text for split in splits),
|
||||
tuple(chain.from_iterable(split.opaque_document_sizes for split in splits)),
|
||||
)
|
||||
|
||||
|
||||
def _split_document_media(part: object) -> _ContentMediaSplit:
|
||||
block: Final = as_str_mapping(part)
|
||||
if block is None or block.get("type") != "document":
|
||||
return _split_media_leaf(part)
|
||||
source: Final = as_str_mapping(block.get("source"))
|
||||
metadata: Final = "".join(value for key in ("title", "context") if isinstance(value := block.get(key), str))
|
||||
if source is None:
|
||||
return _ContentMediaSplit(part, text=metadata, opaque_document_sizes=(0,))
|
||||
if source.get("type") == "text":
|
||||
text: Final = source.get("data")
|
||||
return _ContentMediaSplit(part, text=metadata + (text if isinstance(text, str) else ""))
|
||||
if source.get("type") != "content":
|
||||
return _ContentMediaSplit(part, text=metadata, opaque_document_sizes=(_opaque_document_size(source),))
|
||||
split: Final = _split_media_content(source.get("content"), _split_media_leaf)
|
||||
sanitized: Final = {**block, "source": {**source, "content": split.content}} if split.media else part
|
||||
return _ContentMediaSplit(sanitized, split.media, metadata + split.text, split.opaque_document_sizes)
|
||||
|
||||
|
||||
def _split_tool_result_media(part: object) -> _ContentMediaSplit:
|
||||
block: Final = as_str_mapping(part)
|
||||
if block is None or block.get("type") != "tool_result":
|
||||
return _split_document_media(part)
|
||||
split: Final = _split_media_content(block.get("content"), _split_document_media)
|
||||
sanitized: Final = {**block, "content": split.content} if split.media else part
|
||||
return _ContentMediaSplit(sanitized, split.media, split.text, split.opaque_document_sizes)
|
||||
|
||||
|
||||
def _split_message_content(message: object) -> _ContentMediaSplit:
|
||||
fields: Final = as_str_mapping(message)
|
||||
if fields is None:
|
||||
return _ContentMediaSplit(message)
|
||||
split: Final = _split_media_content(fields.get("content"), _split_tool_result_media)
|
||||
sanitized: Final = {**fields, "content": split.content} if split.media else message
|
||||
return _ContentMediaSplit(sanitized, split.media, split.text, split.opaque_document_sizes)
|
||||
|
||||
|
||||
def split_message_media(messages: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
"""Split declared media from content without mutation, following at most tool-result and document nesting."""
|
||||
parts: Final = _message_content_parts(messages)
|
||||
if parts is None:
|
||||
return messages, ()
|
||||
splits: Final = tuple(map(_split_message_content, parts))
|
||||
media: Final = tuple(chain.from_iterable(split.media for split in splits))
|
||||
if not media:
|
||||
return messages, ()
|
||||
sanitized: Final = tuple(split.content for split in splits)
|
||||
return (sanitized if isinstance(messages, tuple) else list(sanitized)), media
|
||||
|
||||
|
||||
def _message_tool_argument_text(message: Mapping[str, object]) -> str:
|
||||
if message.get("role") != "assistant":
|
||||
return ""
|
||||
functions: Final = chain(
|
||||
(message.get("function_call"),),
|
||||
(
|
||||
call_fields.get("function")
|
||||
for call in _message_content_parts(message.get("tool_calls")) or ()
|
||||
if (call_fields := as_str_mapping(call)) is not None
|
||||
),
|
||||
)
|
||||
return "".join(
|
||||
arguments
|
||||
for function in functions
|
||||
if (function_fields := as_str_mapping(function)) is not None
|
||||
and isinstance(arguments := function_fields.get("arguments"), str)
|
||||
)
|
||||
|
||||
|
||||
def get_token_count_text_from_messages(messages: object, *, include_tool_arguments: bool = False) -> str:
|
||||
"""Visible message content, excluding media bytes and opaque document/reasoning payloads."""
|
||||
text, _ = get_token_count_text_and_document_sizes(messages, include_tool_arguments=include_tool_arguments)
|
||||
return text
|
||||
|
||||
|
||||
def get_token_count_text_and_document_sizes(
|
||||
messages: object, *, include_tool_arguments: bool = False
|
||||
) -> tuple[str, tuple[int, ...]]:
|
||||
"""Visible text and serialized opaque payload sizes, without decoding or fetching documents."""
|
||||
fields: Final = tuple(
|
||||
message for value in _message_content_parts(messages) or () if (message := as_str_mapping(value)) is not None
|
||||
)
|
||||
splits: Final = tuple(_split_message_content(message) for message in fields)
|
||||
text: Final = "".join(
|
||||
split.text
|
||||
+ extract_search_results_text(message.get("search_results"))
|
||||
+ (_message_tool_argument_text(message) if include_tool_arguments else "")
|
||||
for message, split in zip(fields, splits)
|
||||
)
|
||||
return text, tuple(chain.from_iterable(split.opaque_document_sizes for split in splits))
|
||||
|
||||
|
||||
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
|
||||
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
|
||||
|
|
|
|||
|
|
@ -3,15 +3,18 @@
|
|||
import base64
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from itertools import accumulate
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
||||
import anyio
|
||||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
import litellm
|
||||
|
|
@ -35,6 +38,7 @@ from litellm.constants import (
|
|||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
|
||||
ChatTemplateToolSection,
|
||||
chat_template_tool_section,
|
||||
|
|
@ -59,7 +63,6 @@ from litellm.types.llms.anthropic import (
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionDocumentObject,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolParam,
|
||||
|
|
@ -184,22 +187,22 @@ def resize_image_high_res(
|
|||
|
||||
# Test the function with the given example
|
||||
def calculate_tiles_needed(
|
||||
resized_width,
|
||||
resized_height,
|
||||
tile_width=MAX_TILE_WIDTH,
|
||||
tile_height=MAX_TILE_HEIGHT,
|
||||
):
|
||||
resized_width: int,
|
||||
resized_height: int,
|
||||
tile_width: int = MAX_TILE_WIDTH,
|
||||
tile_height: int = MAX_TILE_HEIGHT,
|
||||
) -> int:
|
||||
tiles_across: Final = (resized_width + tile_width - 1) // tile_width
|
||||
tiles_down: Final = (resized_height + tile_height - 1) // tile_height
|
||||
total_tiles: Final = tiles_across * tiles_down
|
||||
return total_tiles
|
||||
|
||||
|
||||
def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int:
|
||||
def high_detail_image_token_upper_bound(base_tokens: int = 85, tile_tokens: int | None = None) -> int:
|
||||
largest_tile_count: Final = calculate_tiles_needed(
|
||||
MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES
|
||||
)
|
||||
return base_tokens + (base_tokens * 2) * largest_tile_count
|
||||
return base_tokens + (base_tokens * 2 if tile_tokens is None else tile_tokens) * largest_tile_count
|
||||
|
||||
|
||||
def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]:
|
||||
|
|
@ -261,6 +264,10 @@ def get_image_dimensions(
|
|||
_header, encoded = data.split(",", 1)
|
||||
img_data = base64.b64decode(encoded)
|
||||
|
||||
return _get_image_dimensions_from_bytes(img_data) or (DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT)
|
||||
|
||||
|
||||
def _get_image_dimensions_from_bytes(img_data: bytes) -> tuple[int, int] | None:
|
||||
img_type: Final = get_image_type(img_data)
|
||||
|
||||
if img_type == "png":
|
||||
|
|
@ -303,8 +310,7 @@ def get_image_dimensions(
|
|||
h = ((bits >> 14) & 0x3FFF) + 1
|
||||
return w, h
|
||||
|
||||
# return sensible default image dimensions if unable to get dimensions
|
||||
return DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT
|
||||
return None
|
||||
|
||||
|
||||
def calculate_img_tokens(
|
||||
|
|
@ -342,6 +348,161 @@ def calculate_img_tokens(
|
|||
return total_tokens
|
||||
|
||||
|
||||
_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
|
||||
_MAX_IMAGE_PATCHES: Final = 30_000
|
||||
_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
|
||||
{
|
||||
"gpt-4o": (85, 170),
|
||||
"gpt-4.1": (85, 170),
|
||||
"gpt-4o-mini": (2833, 5667),
|
||||
"gpt-5": (70, 140),
|
||||
"gpt-5.1": (70, 140),
|
||||
"o1": (75, 150),
|
||||
"o1-pro": (75, 150),
|
||||
"o3": (75, 150),
|
||||
}
|
||||
)
|
||||
_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
|
||||
{
|
||||
"gpt-6-astra": 1.2,
|
||||
"gpt-5.6-sol": 1.2,
|
||||
"gpt-5.6-terra": 1.2,
|
||||
"gpt-5.6-luna": 1.2,
|
||||
"gpt-5.5": 1.2,
|
||||
"gpt-5.4": 1.2,
|
||||
"gpt-5.4-mini": 1.2,
|
||||
"gpt-5.4-nano": 1.2,
|
||||
"gpt-5.2": 1.2,
|
||||
"gpt-4.1-mini": 1.62,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
|
||||
separator: Final = data.find(",", 0, 256)
|
||||
# Adapters can declare the image MIME separately from a generic data URL.
|
||||
# The bounded byte parser still requires a supported image signature.
|
||||
if not data.startswith("data:") or separator < 0 or not data[:separator].endswith(";base64"):
|
||||
return None
|
||||
encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
|
||||
try:
|
||||
header: Final = base64.b64decode(encoded_header, validate=True)
|
||||
dimensions: Final = _get_image_dimensions_from_bytes(header)
|
||||
except (ValueError, TypeError, struct.error):
|
||||
return None
|
||||
if dimensions is None:
|
||||
return None
|
||||
width, height = dimensions
|
||||
return (width, height) if width > 0 and height > 0 else None
|
||||
|
||||
|
||||
def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
|
||||
if model in ("gpt-5.2", "gpt-4.1-mini"):
|
||||
return None if detail == "original" else (2048, 6144)
|
||||
original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
|
||||
effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
|
||||
if effective_detail == "low":
|
||||
return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
|
||||
if effective_detail == "high":
|
||||
return (65_535 if model == "gpt-6-astra" else 2048, 2500)
|
||||
return (
|
||||
(65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
|
||||
)
|
||||
|
||||
|
||||
def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
|
||||
dimension_scale: Final = min(1.0, max_dimension / max(width, height))
|
||||
fitted_width: Final = max(1, round(width * dimension_scale))
|
||||
fitted_height: Final = max(1, round(height * dimension_scale))
|
||||
patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
|
||||
if budget is None or patches <= budget:
|
||||
return patches
|
||||
shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
|
||||
width_patches: Final = fitted_width * shrink / 32
|
||||
height_patches: Final = fitted_height * shrink / 32
|
||||
aligned_shrink: Final = shrink * min(
|
||||
math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
|
||||
)
|
||||
resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
|
||||
resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
|
||||
return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
|
||||
|
||||
|
||||
def _image_policy_model_name(model: str) -> str | None:
|
||||
provider, separator, provider_model = model.partition("/")
|
||||
if not separator:
|
||||
return model
|
||||
if provider == "openai":
|
||||
return provider_model
|
||||
if provider == "openrouter":
|
||||
return provider_model.removeprefix("openai/") if provider_model.startswith("openai/") else None
|
||||
if provider not in ("azure", "azure_ai"):
|
||||
return None
|
||||
catalog_entry: Final = as_str_mapping(
|
||||
cast( # cast-ok: read-only view of the legacy string-keyed registry; as_str_mapping checks the entry
|
||||
Mapping[str, object], litellm.model_cost
|
||||
).get(model)
|
||||
)
|
||||
if catalog_entry is not None and catalog_entry.get("litellm_provider") in ("azure", "azure_ai"):
|
||||
return provider_model.rsplit("/", 1)[-1]
|
||||
return provider_model.removeprefix("responses/").removeprefix("o_series/").removeprefix("openai/")
|
||||
|
||||
|
||||
def _image_model_family(model: str) -> str | None:
|
||||
provider_model: Final = _image_policy_model_name(model)
|
||||
if provider_model is None:
|
||||
return None
|
||||
family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
|
||||
canonical: Final = litellm.azure_llms.get(f"azure/{family}", family) if model.startswith("azure/") else family
|
||||
return canonical if canonical in _TILE_IMAGE_TOKEN_COSTS or canonical in _PATCH_IMAGE_MULTIPLIERS else None
|
||||
|
||||
|
||||
def _image_policy_tokens(family: str | None, dimensions: tuple[int, int] | None, detail: str) -> int:
|
||||
if family is None:
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
|
||||
multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
|
||||
if tile_costs is not None:
|
||||
base_tokens, tile_tokens = tile_costs
|
||||
if detail == "original":
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
if detail == "low":
|
||||
return base_tokens
|
||||
if dimensions is None:
|
||||
return high_detail_image_token_upper_bound(base_tokens, tile_tokens)
|
||||
width, height = dimensions
|
||||
scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
|
||||
tiles: Final = calculate_tiles_needed(max(1, math.floor(width * scale)), max(1, math.floor(height * scale)))
|
||||
return base_tokens + tiles * tile_tokens
|
||||
limits: Final = _patch_image_limits(family, detail)
|
||||
if limits is None or multiplier is None:
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
max_dimension, patch_budget = limits
|
||||
patches: Final = (
|
||||
min(math.ceil(max_dimension / 32) ** 2, patch_budget if patch_budget is not None else _MAX_IMAGE_PATCHES)
|
||||
if dimensions is None
|
||||
else _image_patch_count(*dimensions, max_dimension, patch_budget)
|
||||
)
|
||||
return math.ceil(patches * multiplier)
|
||||
|
||||
|
||||
def estimate_image_tokens(
|
||||
model: str | Sequence[str], data: str, detail: str = "auto", *, fallback_model: str | None = None
|
||||
) -> int:
|
||||
"""Estimate image policies from at most 48 KiB of inline headers, without fetching media."""
|
||||
if detail not in ("low", "high", "auto", "original"):
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
models: Final = (model,) if isinstance(model, str) else model
|
||||
fallback_family: Final = _image_model_family(fallback_model) if fallback_model is not None else None
|
||||
families: Final = tuple(
|
||||
dict.fromkeys(_image_model_family(candidate) or fallback_family for candidate in models or ("",))
|
||||
)
|
||||
if all(family is None for family in families):
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
dimensions: Final = _inline_image_dimensions(data)
|
||||
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
|
||||
|
||||
|
||||
TokenCounterFunction = Callable[[str], int]
|
||||
"""
|
||||
Type for a function that counts tokens in a string.
|
||||
|
|
@ -351,6 +512,7 @@ EXTRAPOLATION_SAMPLES: Final = 16
|
|||
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
|
||||
T_Retval = TypeVar("T_Retval")
|
||||
_COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
|
||||
_TOOL_CHOICE_FUNCTION: Final = TypeAdapter(Mapping[str, object])
|
||||
_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
|
||||
("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
|
||||
)
|
||||
|
|
@ -418,10 +580,10 @@ def token_counter(
|
|||
model="",
|
||||
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
|
||||
text: str | list[str] | None = None,
|
||||
messages: Sequence[AllMessageValues | Message] | None = None,
|
||||
messages: Sequence[object] | None = None,
|
||||
count_response_tokens: bool | None = False,
|
||||
tools: list[ChatCompletionToolParam] | None = None,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
|
||||
tools: Sequence[object] | None = None,
|
||||
tool_choice: object = None,
|
||||
use_default_image_token_count: bool | None = False,
|
||||
default_token_count: int | None = None,
|
||||
) -> int:
|
||||
|
|
@ -600,8 +762,8 @@ def _count_messages(
|
|||
|
||||
def _count_extra(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None,
|
||||
tools: Sequence[object] | None,
|
||||
tool_choice: object,
|
||||
includes_system_message: bool,
|
||||
tool_section: ChatTemplateToolSection | None,
|
||||
) -> int:
|
||||
|
|
@ -638,12 +800,13 @@ def _count_tool_definitions(
|
|||
|
||||
def _count_tool_choice(
|
||||
count_function: TokenCounterFunction,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None,
|
||||
tool_choice: object,
|
||||
) -> int:
|
||||
if tool_choice == "none":
|
||||
return 1
|
||||
if isinstance(tool_choice, dict):
|
||||
return 7 + count_function(str(tool_choice["function"]["name"]))
|
||||
function: Final = _TOOL_CHOICE_FUNCTION.validate_python(tool_choice["function"])
|
||||
return 7 + count_function(str(function["name"]))
|
||||
return 0
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -42,14 +42,22 @@ from litellm.caching.redis_batch import (
|
|||
active_request_redis_batch,
|
||||
)
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.constants import (
|
||||
DEFAULT_IMAGE_TOKEN_COUNT,
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
as_openai_image_part,
|
||||
get_token_count_text_and_document_sizes,
|
||||
get_token_count_text_from_messages,
|
||||
split_message_media,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import normalized_tokens
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
|
|
@ -73,7 +81,6 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
canonical_provider_batch_id,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
|
|
@ -98,6 +105,7 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.router import Router
|
||||
from litellm.types.agents import AgentResponse
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
|
||||
|
|
@ -109,6 +117,9 @@ else:
|
|||
|
||||
|
||||
_REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object])
|
||||
_TOKEN_COUNT_ITEMS: Final = TypeAdapter(list[object])
|
||||
_TOKEN_COUNT_SEQUENCE: Final = TypeAdapter[Sequence[object] | None](Sequence[object] | None)
|
||||
_TOKEN_COUNT_TEXT: Final = TypeAdapter[str | list[str] | None](str | list[str] | None)
|
||||
_MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None)
|
||||
|
||||
|
||||
|
|
@ -135,6 +146,24 @@ class RateLimitedModel:
|
|||
return requested_limit if requested_limit is not None else limits.get(self.group)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RequestInputTokenEstimate:
|
||||
cheap_tokens: int
|
||||
precise_tokens: int | None = None
|
||||
|
||||
|
||||
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
if router is None:
|
||||
return (model,)
|
||||
return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,)
|
||||
|
||||
|
||||
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return _resolve_media_models(llm_router, model, team_id)
|
||||
|
||||
|
||||
def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
|
|
@ -502,18 +531,6 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
|
|||
CallTypes.agenerate_content_stream.value,
|
||||
)
|
||||
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
|
||||
# litellm.token_counter has no per-type handling for "input_audio" content
|
||||
# blocks (unlike images, which use use_default_image_token_count) -- it
|
||||
# silently contributes 0 tokens for them. When the block carries a base64
|
||||
# payload, the estimate is derived from the decoded byte count; when the
|
||||
# block is a reference without a payload (or the payload is missing), this
|
||||
# flat per-block floor is used instead.
|
||||
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
|
||||
# Conservative bytes-per-token assumption for size-based audio estimation:
|
||||
# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing
|
||||
# the lowest reasonable bitrate means we never under-reserve for higher-
|
||||
# quality audio recorded at the same wall-clock duration.
|
||||
_AUDIO_BYTES_PER_TOKEN: Final = 1600
|
||||
# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from
|
||||
# "model_per_project" (the combined-TPM descriptor) so both can be enforced
|
||||
# on the same project+model simultaneously without colliding on cache keys.
|
||||
|
|
@ -867,11 +884,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
] = _resolve_ptu_team_ceiling_via_proxy_router,
|
||||
fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
|
||||
force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
|
||||
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
|
||||
):
|
||||
self.internal_usage_cache = internal_usage_cache
|
||||
self._time_provider = time_provider or datetime.now
|
||||
self._tag_rate_limit_resolver = tag_rate_limit_resolver
|
||||
self._model_group_resolver = model_group_resolver
|
||||
self._media_model_resolver = media_model_resolver
|
||||
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
|
||||
self._fail_closed_resolver = fail_closed_resolver
|
||||
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
|
||||
|
|
@ -1138,6 +1157,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit: int | None = None,
|
||||
call_type: str | None = None,
|
||||
configured_output_tokens: int | None = None,
|
||||
input_estimate: RequestInputTokenEstimate | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Estimate total tokens this request will consume so we can reserve them
|
||||
|
|
@ -1159,11 +1179,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
floor entirely, so the reservation reflects what this tenant's model
|
||||
actually emits rather than one constant shared by every tenant.
|
||||
"""
|
||||
measured_input: Final = (
|
||||
input_estimate
|
||||
if input_estimate is not None
|
||||
else self._estimate_request_input_tokens(_REQUEST_RATE_LIMIT_DATA.validate_python(data), model, call_type)
|
||||
)
|
||||
estimated_input_tokens, max_tokens_estimate = self._estimate_input_and_output_tokens(
|
||||
data=data,
|
||||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
input_estimate=measured_input,
|
||||
)
|
||||
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
|
||||
|
||||
|
|
@ -1184,6 +1210,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type: str | None,
|
||||
configured_output_tokens: int | None,
|
||||
raw_estimate: int,
|
||||
input_estimate: RequestInputTokenEstimate | None = None,
|
||||
) -> int:
|
||||
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
|
||||
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
|
||||
|
|
@ -1194,6 +1221,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
input_estimate=input_estimate,
|
||||
)
|
||||
normalized: Final = normalized_tokens(
|
||||
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
|
||||
|
|
@ -1206,6 +1234,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit: int | None = None,
|
||||
call_type: str | None = None,
|
||||
configured_output_tokens: int | None = None,
|
||||
input_estimate: RequestInputTokenEstimate | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Estimate input tokens and output (max_tokens) budget separately, so
|
||||
|
|
@ -1231,43 +1260,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
"""
|
||||
if not isinstance(data, dict):
|
||||
return 0, 0
|
||||
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
|
||||
estimable_data: Final = translated_data if translated_data is not None else data
|
||||
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
|
||||
(None, None, estimable_data.get("input"))
|
||||
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
|
||||
else (None, estimable_data.get("prompt"), None)
|
||||
if call_type in TEXT_COMPLETION_API_CALL_TYPES
|
||||
else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(estimable_data)), None)
|
||||
if call_type in EXTRACTION_API_CALL_TYPES
|
||||
else (estimable_data.get("messages"), None, None)
|
||||
if call_type
|
||||
else (
|
||||
estimable_data.get("messages"),
|
||||
estimable_data.get("prompt"),
|
||||
estimable_data.get("input"),
|
||||
estimated_input_tokens: Final = (
|
||||
input_estimate
|
||||
if input_estimate is not None
|
||||
else self._estimate_request_input_tokens(
|
||||
_REQUEST_RATE_LIMIT_DATA.validate_python(data), call_type=call_type
|
||||
)
|
||||
)
|
||||
messages, prompt, raw_input = selected_fields
|
||||
input_text: Final = (
|
||||
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
|
||||
)
|
||||
|
||||
total_chars: Final = (
|
||||
len(get_str_from_messages(messages))
|
||||
if isinstance(messages, list) and messages
|
||||
else len(prompt)
|
||||
if isinstance(prompt, str)
|
||||
else sum(len(str(item)) for item in prompt)
|
||||
if isinstance(prompt, list)
|
||||
else len(input_text)
|
||||
if isinstance(input_text, str)
|
||||
else sum(len(str(item)) for item in input_text)
|
||||
if isinstance(input_text, list)
|
||||
else 0
|
||||
)
|
||||
|
||||
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
|
||||
).cheap_tokens
|
||||
|
||||
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
|
||||
generates_no_output: Final = (
|
||||
|
|
@ -1283,7 +1282,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
max_tokens_estimate: Final = (
|
||||
0
|
||||
if generates_no_output
|
||||
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
|
||||
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
|
||||
else explicit_max_tokens
|
||||
if explicit_max_tokens is not None
|
||||
else configured_output_tokens
|
||||
|
|
@ -3753,72 +3752,37 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_audio_block_tokens(block: object) -> int:
|
||||
"""
|
||||
Token estimate for one ``input_audio`` content block.
|
||||
|
||||
When the block carries a base64 ``data`` payload, the estimate comes
|
||||
from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``),
|
||||
assuming the lowest reasonable audio bitrate so we never under-reserve
|
||||
for higher-quality recordings of the same duration.
|
||||
|
||||
When no payload is present (reference-only block or missing ``data``),
|
||||
falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``.
|
||||
"""
|
||||
if not isinstance(block, dict):
|
||||
return DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
input_audio: Final = block.get("input_audio")
|
||||
b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
|
||||
if b64_data and isinstance(b64_data, str):
|
||||
decoded_bytes: Final = len(b64_data) * 3 // 4
|
||||
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
|
||||
return DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
|
||||
@classmethod
|
||||
def _estimate_audio_content_tokens(cls, messages: object) -> int:
|
||||
"""
|
||||
Sum of per-block audio token estimates across all ``messages``.
|
||||
Returns 0 when there are no ``input_audio`` blocks, which the caller
|
||||
uses to skip the (relatively expensive) strip pass.
|
||||
"""
|
||||
if not isinstance(messages, list):
|
||||
return 0
|
||||
return sum(
|
||||
cls._estimate_audio_block_tokens(block)
|
||||
for message in messages
|
||||
if isinstance(message, dict)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, list)
|
||||
for block in content
|
||||
if isinstance(block, dict) and block.get("type") == "input_audio"
|
||||
def _estimate_image_block_tokens(
|
||||
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
|
||||
) -> int:
|
||||
normalized: Final = as_openai_image_part(block)
|
||||
image: Final = _callback_kwargs_mapping(normalized.get("image_url") if normalized is not None else None)
|
||||
url: Final = image.get("url")
|
||||
detail: Final = _callback_kwargs_mapping(block.get("image_url")).get("detail") or block.get("detail")
|
||||
return estimate_image_tokens(
|
||||
models,
|
||||
url if isinstance(url, str) else "",
|
||||
detail if isinstance(detail, str) else "auto",
|
||||
fallback_model=fallback_model,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _strip_audio_content_blocks(messages: object) -> object:
|
||||
"""
|
||||
Drop ``input_audio`` content blocks before passing ``messages`` to
|
||||
``token_counter``, which raises ``ValueError`` on them (no per-type
|
||||
handling, unlike images). The audio contribution is added back
|
||||
separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the
|
||||
message (text/images/tools) still gets counted accurately instead of
|
||||
the whole call falling back to the cheap char-count estimate.
|
||||
"""
|
||||
if not isinstance(messages, list):
|
||||
return messages
|
||||
sanitized: Final[list[object]] = [] # mutable-ok: token_counter requires a list of message dicts
|
||||
for message in messages:
|
||||
if not isinstance(message, dict):
|
||||
sanitized.append(message)
|
||||
continue
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
sanitized.append(message)
|
||||
continue
|
||||
filtered_content = [
|
||||
block for block in content if not (isinstance(block, dict) and block.get("type") == "input_audio")
|
||||
]
|
||||
sanitized.append({**message, "content": filtered_content})
|
||||
return sanitized
|
||||
def _estimate_audio_block_tokens(
|
||||
block: object, models: tuple[str, ...] = (), fallback_model: str | None = None
|
||||
) -> int:
|
||||
audio: Final = _callback_kwargs_mapping(_callback_kwargs_mapping(block).get("input_audio"))
|
||||
data: Final = audio.get("data")
|
||||
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
|
||||
|
||||
@staticmethod
|
||||
def _requires_input_count_offload(data: Mapping[str, object], call_type: str | None) -> bool:
|
||||
if call_type in RESPONSES_API_CALL_TYPES + GOOGLE_GENAI_NATIVE_CALL_TYPES:
|
||||
return True
|
||||
messages: Final = data.get("messages")
|
||||
return isinstance(messages, list) and any(
|
||||
isinstance(_callback_kwargs_mapping(message).get("content"), list)
|
||||
for message in _TOKEN_COUNT_ITEMS.validate_python(messages)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _responses_input_to_chat_messages(data: object) -> Sequence[object]:
|
||||
|
|
@ -3873,93 +3837,110 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
state: Final = data.get("state")
|
||||
return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}"
|
||||
|
||||
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
|
||||
"""
|
||||
Model-aware input token estimate for the project ITPM reservation,
|
||||
using ``litellm.token_counter`` -- the same approach the
|
||||
deployment-level itpm/otpm check uses in
|
||||
``io_token_rate_limit_check.py``. Unlike the cheap char-count
|
||||
estimate the combined-TPM path uses, this accounts for image/tool
|
||||
content and derives per-``input_audio``-block estimates from the
|
||||
base64 payload size (assuming the lowest reasonable bitrate so
|
||||
longer recordings always reserve proportionally more), so a burst
|
||||
of multimodal, tool-heavy, or audio-heavy requests can't each
|
||||
reserve only the one-token floor and blow past ITPM before
|
||||
post-call reconciliation catches up.
|
||||
|
||||
For the Responses API, ``input`` is converted to chat messages first
|
||||
(via ``_responses_input_to_chat_messages``) so its own multimodal
|
||||
content blocks are counted the same way; ``token_counter``'s ``text``
|
||||
argument can only see plain strings in a list, not content blocks.
|
||||
|
||||
Falls back to the cheap char-count estimate if ``token_counter``
|
||||
can't resolve a tokenizer for this model (e.g. an unrecognized
|
||||
custom model name) or otherwise raises -- the audio add-on still
|
||||
applies on top of the fallback.
|
||||
"""
|
||||
def _estimate_request_input_tokens(
|
||||
self,
|
||||
data: object,
|
||||
model: str | None = None,
|
||||
call_type: str | None = None,
|
||||
models: tuple[str, ...] = (),
|
||||
precise: bool = False,
|
||||
) -> RequestInputTokenEstimate:
|
||||
from litellm import token_counter
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return 0
|
||||
if not isinstance(data, Mapping):
|
||||
return RequestInputTokenEstimate(0, 0 if precise else None)
|
||||
request: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(data)
|
||||
is_responses_request: Final = call_type in RESPONSES_API_CALL_TYPES
|
||||
translated_request: Final = (
|
||||
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
|
||||
translated: Final = (
|
||||
None if is_responses_request else self._translate_google_genai_native_request(request, call_type)
|
||||
)
|
||||
is_embedding_request: Final = self._is_embedding_request(data, call_type)
|
||||
embedding_text: Final = data.get("input") if is_embedding_request else None
|
||||
pretokenized_input_tokens: Final = (
|
||||
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
|
||||
)
|
||||
if pretokenized_input_tokens is not None:
|
||||
return pretokenized_input_tokens
|
||||
embedding_input: Final = request.get("input") if self._is_embedding_request(request, call_type) else None
|
||||
pretokenized_tokens: Final = self._count_pretokenized_embedding_input(embedding_input)
|
||||
is_rerank_request: Final = call_type in RERANK_API_CALL_TYPES
|
||||
|
||||
prompt: Final = data.get("prompt")
|
||||
fallback_text: Final = prompt if prompt is not None else data.get("input")
|
||||
selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = (
|
||||
(self._responses_input_to_chat_messages(data), None, data.get("tools"), data.get("tool_choice"))
|
||||
prompt: Final = request.get("prompt")
|
||||
selected: Final[tuple[object, object, object, object]] = (
|
||||
(self._responses_input_to_chat_messages(request), None, request.get("tools"), request.get("tool_choice"))
|
||||
if is_responses_request
|
||||
else (
|
||||
translated_request.get("messages"),
|
||||
None,
|
||||
translated_request.get("tools"),
|
||||
translated_request.get("tool_choice"),
|
||||
)
|
||||
if translated_request is not None
|
||||
else (None, embedding_text, data.get("tools"), data.get("tool_choice"))
|
||||
if is_embedding_request
|
||||
else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice"))
|
||||
if call_type in RERANK_API_CALL_TYPES
|
||||
else (None, prompt, data.get("tools"), data.get("tool_choice"))
|
||||
else (translated.get("messages"), None, translated.get("tools"), translated.get("tool_choice"))
|
||||
if translated is not None
|
||||
else (None, embedding_input, request.get("tools"), request.get("tool_choice"))
|
||||
if self._is_embedding_request(request, call_type)
|
||||
else (None, self._rerank_input_to_text(request), request.get("tools"), request.get("tool_choice"))
|
||||
if is_rerank_request
|
||||
else (None, prompt, request.get("tools"), request.get("tool_choice"))
|
||||
if call_type in TEXT_COMPLETION_API_CALL_TYPES
|
||||
else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(data)), None, None)
|
||||
else (None, self._extraction_input_text(request), None, None)
|
||||
if call_type in EXTRACTION_API_CALL_TYPES
|
||||
else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice"))
|
||||
)
|
||||
messages, selected_text, countable_tools, countable_tool_choice = selected_inputs
|
||||
|
||||
audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
|
||||
countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
|
||||
|
||||
try:
|
||||
estimate: Final = max(
|
||||
0,
|
||||
int(
|
||||
token_counter(
|
||||
model=model or "",
|
||||
messages=countable_messages,
|
||||
text=selected_text,
|
||||
tools=countable_tools,
|
||||
tool_choice=countable_tool_choice,
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
),
|
||||
else (
|
||||
request.get("messages"),
|
||||
None if call_type else prompt if prompt is not None else request.get("input"),
|
||||
request.get("tools"),
|
||||
request.get("tool_choice"),
|
||||
)
|
||||
return estimate + audio_token_estimate
|
||||
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
|
||||
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
|
||||
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
|
||||
return estimated_input_tokens + audio_token_estimate
|
||||
)
|
||||
messages, text, tools, tool_choice = selected
|
||||
countable_messages, media = split_message_media(messages)
|
||||
model_value: Final = request.get("model")
|
||||
fallback_model: Final = model if model is not None else model_value if isinstance(model_value, str) else None
|
||||
media_tokens: Final = sum(
|
||||
self._estimate_audio_block_tokens(block, models, fallback_model)
|
||||
if block.get("type") == "input_audio"
|
||||
else self._estimate_image_block_tokens(block, models, fallback_model)
|
||||
for block in media
|
||||
)
|
||||
message_text, opaque_document_sizes = get_token_count_text_and_document_sizes(
|
||||
countable_messages, include_tool_arguments=is_responses_request
|
||||
)
|
||||
text_chars: Final = (
|
||||
len(message_text)
|
||||
if isinstance(messages, list) and messages
|
||||
else len(text)
|
||||
if isinstance(text, str)
|
||||
else sum(len(str(item)) for item in _TOKEN_COUNT_ITEMS.validate_python(text))
|
||||
if isinstance(text, list)
|
||||
else 0
|
||||
)
|
||||
cheap_chars: Final = (
|
||||
len(get_token_count_text_from_messages(request.get("messages"))) if is_rerank_request else text_chars
|
||||
)
|
||||
# Keep opaque bytes' character fallback until precise counting can read their pages.
|
||||
cheap_tokens: Final = (
|
||||
(max(1, cheap_chars // DEFAULT_CHARS_PER_TOKEN) if cheap_chars > 0 else 0)
|
||||
+ media_tokens
|
||||
+ sum(
|
||||
max(DEFAULT_IMAGE_TOKEN_COUNT, (size + DEFAULT_CHARS_PER_TOKEN - 1) // DEFAULT_CHARS_PER_TOKEN)
|
||||
for size in opaque_document_sizes
|
||||
)
|
||||
)
|
||||
if not precise:
|
||||
return RequestInputTokenEstimate(cheap_tokens)
|
||||
if pretokenized_tokens is not None:
|
||||
return RequestInputTokenEstimate(cheap_tokens, pretokenized_tokens)
|
||||
try:
|
||||
precise_tokens: Final = (
|
||||
max(
|
||||
0,
|
||||
int(
|
||||
token_counter(
|
||||
model=fallback_model or "",
|
||||
messages=_TOKEN_COUNT_SEQUENCE.validate_python(countable_messages),
|
||||
text=_TOKEN_COUNT_TEXT.validate_python(text),
|
||||
tools=_TOKEN_COUNT_SEQUENCE.validate_python(tools),
|
||||
tool_choice=tool_choice,
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
+ media_tokens
|
||||
)
|
||||
except Exception: # noqa: BLE001 # tokenizer failures use the already measured input
|
||||
fallback_tokens: Final = text_chars // DEFAULT_CHARS_PER_TOKEN if is_rerank_request else cheap_tokens
|
||||
return RequestInputTokenEstimate(cheap_tokens, fallback_tokens)
|
||||
return RequestInputTokenEstimate(cheap_tokens, precise_tokens)
|
||||
|
||||
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
|
||||
return self._estimate_request_input_tokens(data, model, call_type, precise=True).precise_tokens or 0
|
||||
|
||||
async def _reserve_project_io_tokens_or_raise(
|
||||
self,
|
||||
|
|
@ -3972,6 +3953,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type: str | None = None,
|
||||
endpoint_type: EndpointType = EndpointType.GENERIC,
|
||||
reuses_request_slot: bool = False,
|
||||
input_estimate: RequestInputTokenEstimate | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style
|
||||
|
|
@ -3999,15 +3981,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if v is not None
|
||||
]
|
||||
min_configured_otpm_limit: Final = min(configured_otpm_limits) if configured_otpm_limits else None
|
||||
measured_input: Final = (
|
||||
input_estimate
|
||||
if input_estimate is not None
|
||||
else await offload_token_count(self._estimate_request_input_tokens)(
|
||||
data=_REQUEST_RATE_LIMIT_DATA.validate_python(data),
|
||||
model=requested_model,
|
||||
call_type=call_type,
|
||||
models=self._media_model_resolver(requested_model or "", user_api_key_dict.team_id),
|
||||
precise=any(d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY for d in io_token_descriptors),
|
||||
)
|
||||
)
|
||||
_, raw_estimated_output_tokens = self._estimate_input_and_output_tokens(
|
||||
data=data,
|
||||
min_configured_tpm_limit=min_configured_otpm_limit,
|
||||
call_type=call_type,
|
||||
input_estimate=measured_input,
|
||||
)
|
||||
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=requested_model, call_type=call_type
|
||||
estimated_input_tokens: Final = max(
|
||||
measured_input.precise_tokens if measured_input.precise_tokens is not None else measured_input.cheap_tokens,
|
||||
1,
|
||||
)
|
||||
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
|
||||
estimated_output_tokens: Final = (
|
||||
raw_estimated_output_tokens
|
||||
if self._has_explicit_output_cap(data, call_type)
|
||||
|
|
@ -4323,6 +4317,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
has_tpm_limits: Final = bool(configured_tpm_limits)
|
||||
|
||||
needs_precise_input: Final = any(d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY for d in descriptors)
|
||||
needs_input_estimate: Final = (has_tpm_limits and self.tpm_reservation_enabled) or any(
|
||||
d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for d in descriptors
|
||||
)
|
||||
offload_input: Final = self._requires_input_count_offload(request_data, call_type)
|
||||
input_estimate: Final = (
|
||||
await offload_token_count(self._estimate_request_input_tokens)(
|
||||
data=request_data,
|
||||
model=requested_model,
|
||||
call_type=call_type,
|
||||
models=self._media_model_resolver(requested_model or "", user_api_key_dict.team_id)
|
||||
if offload_input
|
||||
else (),
|
||||
precise=needs_precise_input,
|
||||
)
|
||||
if needs_input_estimate and (offload_input or needs_precise_input)
|
||||
else self._estimate_request_input_tokens(request_data, requested_model, call_type)
|
||||
if needs_input_estimate
|
||||
else None
|
||||
)
|
||||
|
||||
# Populated on a successful combined-TPM reservation below, so the
|
||||
# project ITPM/OTPM block further down can roll it back if a
|
||||
# different bucket in the same request subsequently hits its
|
||||
|
|
@ -4366,6 +4381,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
input_estimate=input_estimate,
|
||||
),
|
||||
1,
|
||||
)
|
||||
|
|
@ -4390,6 +4406,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
raw_estimate=estimated_tokens,
|
||||
input_estimate=input_estimate,
|
||||
)
|
||||
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
|
||||
if not reuses_request_slot:
|
||||
|
|
@ -4453,6 +4470,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
tpm_reservation_scopes=tpm_reservation_scopes,
|
||||
tpm_reservation_amount=tpm_reservation_amount,
|
||||
input_estimate=input_estimate,
|
||||
call_type=call_type,
|
||||
endpoint_type=endpoint_type,
|
||||
reuses_request_slot=reuses_request_slot,
|
||||
|
|
|
|||
|
|
@ -10224,28 +10224,38 @@ class Router:
|
|||
model_info[field] = backend_value
|
||||
|
||||
@staticmethod
|
||||
def _cost_map_backend_model(deployment: Deployment) -> str:
|
||||
model_info_base_model: Final = deployment.model_info.base_model
|
||||
def _cost_map_backend_model(deployment: Deployment | DeploymentTypedDict) -> str:
|
||||
model_info_base_model: Final = (
|
||||
deployment.model_info.base_model
|
||||
if isinstance(deployment, Deployment)
|
||||
else _configured_model_info(deployment).get("base_model")
|
||||
)
|
||||
if isinstance(model_info_base_model, str) and model_info_base_model:
|
||||
return model_info_base_model
|
||||
params_base_model: Final = deployment.litellm_params.get("base_model")
|
||||
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(
|
||||
deployment.litellm_params.model_dump(exclude_none=True)
|
||||
if isinstance(deployment, Deployment)
|
||||
else deployment["litellm_params"]
|
||||
)
|
||||
params_base_model: Final = params.get("base_model")
|
||||
if isinstance(params_base_model, str) and params_base_model:
|
||||
return params_base_model
|
||||
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True))
|
||||
model: Final = (
|
||||
deployment.litellm_params.model
|
||||
if isinstance(deployment, Deployment)
|
||||
else deployment["litellm_params"].get("model", "")
|
||||
)
|
||||
invoked_model: Final = params.get("model_id")
|
||||
if (
|
||||
isinstance(invoked_model, str)
|
||||
and invoked_model
|
||||
and (
|
||||
deployment.litellm_params.custom_llm_provider == "bedrock"
|
||||
or deployment.litellm_params.model.startswith("bedrock/")
|
||||
)
|
||||
and (params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/"))
|
||||
):
|
||||
profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}"
|
||||
profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock")
|
||||
if not isinstance(profile_info, ModelInfoLookupMiss):
|
||||
return profile_model
|
||||
return deployment.litellm_params.model
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def _inherit_builtin_service_tier_pricing(
|
||||
|
|
@ -13795,7 +13805,9 @@ class Router:
|
|||
]
|
||||
return named or self._get_wildcard_deployments(model_name=model_group)
|
||||
|
||||
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
|
||||
def resolved_litellm_models(
|
||||
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
|
||||
) -> tuple[str, ...]:
|
||||
"""The provider model strings `model_name` can actually be served by on this proxy.
|
||||
|
||||
`get_model_list` composes every channel the request path itself uses (exact name,
|
||||
|
|
@ -13808,6 +13820,12 @@ class Router:
|
|||
deployment behind it - so the fallback for an empty result is the caller's policy,
|
||||
never this function's.
|
||||
"""
|
||||
if use_base_model:
|
||||
return tuple(
|
||||
backend_model
|
||||
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
if (backend_model := self._cost_map_backend_model(deployment))
|
||||
)
|
||||
return tuple(
|
||||
litellm_model
|
||||
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
|
|
|
|||
|
|
@ -224,7 +224,6 @@ from litellm.types.llms.openai import (
|
|||
AllMessageValues,
|
||||
AllPromptValues,
|
||||
ChatCompletionAssistantToolCall,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
OpenAITextCompletionUserMessage,
|
||||
|
|
@ -2822,10 +2821,10 @@ def token_counter(
|
|||
model="",
|
||||
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
|
||||
text: str | list[str] | None = None,
|
||||
messages: Sequence | None = None,
|
||||
messages: Sequence[object] | None = None,
|
||||
count_response_tokens: bool | None = False,
|
||||
tools: list[ChatCompletionToolParam] | None = None,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
|
||||
tools: Sequence[object] | None = None,
|
||||
tool_choice: object = None,
|
||||
use_default_image_token_count: bool | None = False,
|
||||
default_token_count: int | None = None,
|
||||
) -> int:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
get_format_from_file_id,
|
||||
get_semantic_cache_prompt_from_messages,
|
||||
get_str_from_messages,
|
||||
get_token_count_text_and_document_sizes,
|
||||
get_token_count_text_from_messages,
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion,
|
||||
hoist_images_from_tool_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
|
|
@ -35,6 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
schema_has_unresolvable_ref,
|
||||
schema_property_count,
|
||||
split_concatenated_json_objects,
|
||||
split_message_media,
|
||||
strip_encrypted_reasoning_from_messages,
|
||||
strip_litellm_response_only_fields,
|
||||
system_messages_first,
|
||||
|
|
@ -2465,6 +2468,169 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch
|
|||
assert schema_grammar_branch_count(schema) == branches
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"media",
|
||||
[
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}},
|
||||
{"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"},
|
||||
{"type": "image", "source": {"type": "file", "file_id": "opaque-image"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}},
|
||||
{"type": "image"},
|
||||
{"type": "input_audio", "input_audio": None},
|
||||
],
|
||||
)
|
||||
def test_split_message_media_preserves_every_nonmedia_sibling_and_media_detail(
|
||||
media: Mapping[str, object],
|
||||
) -> None:
|
||||
siblings: Final = ["visible", None, 7, {"type": []}, {"type": "text", "text": "caption"}]
|
||||
messages: Final = [{"role": "tool", "tool_call_id": "call_1", "content": [*siblings, media, media]}]
|
||||
original: Final = copy.deepcopy(messages)
|
||||
|
||||
sanitized, leaves = split_message_media(messages)
|
||||
|
||||
assert sanitized == [{"role": "tool", "tool_call_id": "call_1", "content": siblings}]
|
||||
assert leaves == (media, media)
|
||||
assert messages == original
|
||||
assert split_message_media(sanitized) == (sanitized, ())
|
||||
|
||||
|
||||
def test_split_message_media_keeps_nested_tool_and_document_text_and_metadata() -> None:
|
||||
direct: Final = {"type": "image_url", "image_url": "https://example.com/direct.png"}
|
||||
image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}}
|
||||
audio: Final = {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}
|
||||
text: Final = {"type": "text", "text": "document text"}
|
||||
file: Final = {"type": "file", "file": {"file_id": "opaque-file"}}
|
||||
opaque: Final = {"type": "document", "source": {"type": "url", "url": "https://example.com/document.pdf"}}
|
||||
document: Final = {
|
||||
"type": "document", "title": "title", "context": "context", "citations": {"enabled": True},
|
||||
"source": {"type": "content", "content": [text, image, file, "tail"]},
|
||||
}
|
||||
tool_result: Final = {
|
||||
"type": "tool_result", "tool_use_id": "toolu_1", "is_error": False,
|
||||
"content": ["tool text", audio, document, opaque],
|
||||
}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": [direct, tool_result]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_2", "content": "result"},
|
||||
{"type": "document", "source": {"type": "content", "content": "document"}},
|
||||
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "declared"}},
|
||||
]},
|
||||
]
|
||||
original: Final = copy.deepcopy(messages)
|
||||
|
||||
sanitized, leaves = split_message_media(messages)
|
||||
|
||||
assert sanitized == [{
|
||||
"role": "user", "content": [{
|
||||
**tool_result,
|
||||
"content": ["tool text", {**document, "source": {"type": "content", "content": [text, file, "tail"]}}, opaque],
|
||||
}],
|
||||
}, messages[1]]
|
||||
assert leaves == (direct, audio, image)
|
||||
expected_text: Final = "tool texttitlecontextdocument texttailresultdocumentdeclared"
|
||||
assert get_token_count_text_from_messages(sanitized) == expected_text
|
||||
visible_text, opaque_sizes = get_token_count_text_and_document_sizes(sanitized)
|
||||
assert visible_text == expected_text
|
||||
assert len(opaque_sizes) == 2 and all(size > 0 for size in opaque_sizes)
|
||||
assert messages == original
|
||||
|
||||
|
||||
def test_split_message_media_does_not_walk_arbitrary_tool_arguments_or_invalid_nesting() -> None:
|
||||
image: Final = {"type": "image", "source": {"type": "url", "url": "https://example.com/image.png"}}
|
||||
document: Final = {"type": "document", "source": {"type": "content", "content": [image]}}
|
||||
tool: Final = {"type": "tool_result", "content": [image]}
|
||||
blocks: Final = [
|
||||
{"type": "tool_use", "name": "inspect", "input": {"content": [image]}},
|
||||
{"type": "unknown", "content": [image]},
|
||||
{"type": "document", "source": {"type": "content", "content": [document]}},
|
||||
{"type": "tool_result", "content": [tool]},
|
||||
]
|
||||
messages: Final = [{"role": "assistant", "content": blocks}]
|
||||
sanitized, leaves = split_message_media(messages)
|
||||
assert sanitized == messages
|
||||
assert leaves == ()
|
||||
|
||||
|
||||
def test_token_count_text_includes_only_opaque_assistant_argument_strings_when_requested() -> None:
|
||||
arguments: Final = '{"type":"input_image","image_url":"data:image/png;base64,IMAGE"}'
|
||||
calls: Final = [
|
||||
{"function": {"arguments": arguments}},
|
||||
{"function": {"arguments": {"text": "do not traverse"}}},
|
||||
{"function": None},
|
||||
None,
|
||||
]
|
||||
messages: Final = [
|
||||
{"role": "assistant", "content": "before", "tool_calls": calls, "function_call": {"arguments": "legacy"}},
|
||||
{"role": "user", "content": "after", "tool_calls": calls},
|
||||
]
|
||||
original: Final = copy.deepcopy(messages)
|
||||
|
||||
assert get_token_count_text_from_messages(messages) == "beforeafter"
|
||||
assert get_token_count_text_from_messages(messages, include_tool_arguments=True) == (
|
||||
"beforelegacy" + arguments + "after"
|
||||
)
|
||||
assert messages == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("part", "expected"),
|
||||
[
|
||||
(
|
||||
{"type": "file", "file": {"file_data": "data:image/png;base64,IMAGE", "detail": "low"}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "low"}},
|
||||
),
|
||||
(
|
||||
{"type": "input_file", "file_data": "data:image/png;base64,IMAGE", "detail": "low"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "low"}},
|
||||
),
|
||||
(
|
||||
{"type": "file", "file": {"file_id": "opaque", "mime_type": "image/jpeg"}},
|
||||
{"type": "image_url", "image_url": {"url": "opaque", "detail": "auto"}},
|
||||
),
|
||||
(
|
||||
{"type": "file", "file": {"file_data": "data:audio/wav;base64,AUDIO"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}},
|
||||
),
|
||||
(
|
||||
{"type": "file", "file": {"file_data": "AUDIO", "content_type": "audio/mpeg"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "mpeg"}},
|
||||
),
|
||||
(
|
||||
{"type": "file", "file": {"file_id": "https://example.com/audio.wav", "format": "audio/wav"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_split_message_media_normalizes_only_explicit_image_and_audio_files(
|
||||
part: Mapping[str, object], expected: Mapping[str, object],
|
||||
) -> None:
|
||||
messages: Final = [{"role": "user", "content": [part]}]
|
||||
assert split_message_media(messages) == ([{"role": "user", "content": []}], (expected,))
|
||||
assert get_token_count_text_and_document_sizes(messages) == ("", ())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"part",
|
||||
[
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,PDF"}},
|
||||
{"type": "file", "file": {"file_data": "data:video/mp4;base64,VIDEO"}},
|
||||
{"type": "file", "file": {"file_id": "https://example.com/name.png"}},
|
||||
{"type": "file", "file": {"file_data": "IMAGE", "format": "application/pdf"}},
|
||||
{"type": "file", "file": None},
|
||||
{"type": "thinking", "thinking": "thought", "signature": "opaque"},
|
||||
{"type": "tool_use", "name": "read", "input": {"path": "file.txt"}},
|
||||
{"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}},
|
||||
],
|
||||
)
|
||||
def test_split_message_media_preserves_opaque_documents_and_unclassified_files(part: Mapping[str, object]) -> None:
|
||||
messages: Final = [{"role": "user", "content": [part]}]
|
||||
sanitized, leaves = split_message_media(messages)
|
||||
assert sanitized == messages
|
||||
assert leaves == ()
|
||||
assert get_token_count_text_from_messages(messages) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("part_type", ["image_url", "input_image"])
|
||||
@pytest.mark.parametrize(
|
||||
"fields",
|
||||
|
|
|
|||
|
|
@ -2,16 +2,22 @@
|
|||
Test the audio utils functionality in litellm_core_utils/audio_utils/utils.py
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Final
|
||||
from unittest.mock import mock_open, patch
|
||||
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
AUDIO_BYTES_PER_TOKEN,
|
||||
DEFAULT_AUDIO_TOKEN_ESTIMATE,
|
||||
ProcessedAudioFile,
|
||||
calculate_request_duration,
|
||||
estimate_audio_tokens,
|
||||
get_audio_file_content_hash,
|
||||
get_audio_file_for_health_check,
|
||||
get_audio_file_name,
|
||||
|
|
@ -19,6 +25,74 @@ from litellm.litellm_core_utils.audio_utils.utils import (
|
|||
)
|
||||
|
||||
|
||||
def _encoded_audio(seconds: int, sample_rate: int, channels: int, container: str) -> str:
|
||||
buffer: Final = io.BytesIO()
|
||||
with sf.SoundFile(buffer, "w", samplerate=sample_rate, channels=channels, format=container) as audio:
|
||||
audio.buffer_write(b"\0" * (seconds * sample_rate * channels * 2), dtype="int16")
|
||||
return base64.b64encode(buffer.getvalue()).decode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sample_rate", "channels", "container"),
|
||||
[
|
||||
(8000, 1, "WAV"),
|
||||
(48000, 2, "WAV"),
|
||||
(24000, 1, "FLAC"),
|
||||
(48000, 2, "OGG"),
|
||||
(24000, 1, "MP3"),
|
||||
],
|
||||
)
|
||||
def test_audio_estimate_uses_duration_across_containers_sample_rates_and_channels(
|
||||
sample_rate: int, channels: int, container: str
|
||||
) -> None:
|
||||
short: Final = _encoded_audio(1, sample_rate, channels, container)
|
||||
long: Final = _encoded_audio(31, sample_rate, channels, container)
|
||||
assert estimate_audio_tokens(short) == 10
|
||||
assert estimate_audio_tokens(long) == 310
|
||||
|
||||
|
||||
# Audio rates verified 2026-10-26: https://ai.google.dev/gemini-api/docs/audio and
|
||||
# https://developers.openai.com/api/docs/guides/voice-latency-cost?voice-api=realtime
|
||||
@pytest.mark.parametrize(
|
||||
("models", "fallback_model", "expected"),
|
||||
[
|
||||
(("gpt-audio-1.5",), None, 10),
|
||||
(("gemini/gemini-3.8-flash",), None, 32),
|
||||
(("vertex_ai/gemini-3.8-flash",), None, 32),
|
||||
(("openrouter/google/gemini-3.8-flash",), None, 32),
|
||||
(("gpt-audio-1.5", "gemini-3.8-flash"), None, 32),
|
||||
(("azure/audio-deployment",), "gpt-audio-1.5", 10),
|
||||
(("opaque-deployment",), "gemini-3.8-flash", 32),
|
||||
(("openai/gpt-audio-1.5",), "gemini-3.8-flash", 10),
|
||||
((), "gemini-3.8-flash", 32),
|
||||
(("other/gemini-3.8-flash",), None, 10),
|
||||
],
|
||||
)
|
||||
def test_audio_estimate_selects_the_backend_rate_before_the_requested_model(
|
||||
models: tuple[str, ...], fallback_model: str | None, expected: int
|
||||
) -> None:
|
||||
assert estimate_audio_tokens(_encoded_audio(1, 8000, 1, "WAV"), models, fallback_model) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data", [None, "", "invalid-base64!", base64.b64encode(b"unreadable audio").decode()])
|
||||
def test_audio_estimate_preserves_a_nonzero_fallback_for_unreadable_audio(data: str | None) -> None:
|
||||
assert estimate_audio_tokens(data) == DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
|
||||
|
||||
def test_audio_estimate_preserves_size_based_fallback_for_large_unreadable_audio() -> None:
|
||||
data: Final = base64.b64encode(b"\0" * (AUDIO_BYTES_PER_TOKEN * 400)).decode()
|
||||
assert estimate_audio_tokens(data) == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_bytes", [0, 1])
|
||||
def test_audio_estimate_bounds_base64_allocation_before_decoding(extra_bytes: int) -> None:
|
||||
data: Final = base64.b64encode(b"\0" * (16 * 1024 * 1024 + extra_bytes)).decode()
|
||||
with patch("base64.b64decode", wraps=base64.b64decode) as decode_audio:
|
||||
estimate: Final = estimate_audio_tokens(data)
|
||||
assert estimate == len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN
|
||||
assert decode_audio.call_count == (1 if extra_bytes == 0 else 0)
|
||||
|
||||
|
||||
class TestProcessAudioFile:
|
||||
"""Test the process_audio_file function with various input types"""
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import (
|
|||
_get_extrapolating_count_function,
|
||||
_get_tiktoken_count_function,
|
||||
calculate_img_tokens,
|
||||
estimate_image_tokens,
|
||||
high_detail_image_token_upper_bound,
|
||||
offload_token_count,
|
||||
)
|
||||
|
|
@ -2025,6 +2026,173 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_
|
|||
assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound()
|
||||
|
||||
|
||||
# Image policies verified 2026-10-24: https://developers.openai.com/api/docs/guides/images-vision
|
||||
@pytest.mark.parametrize(
|
||||
("model", "width", "height", "detail", "expected"),
|
||||
[
|
||||
("gpt-4o", 16, 16, "high", 255),
|
||||
("gpt-4.1", 1024, 1024, "high", 765),
|
||||
("gpt-4o-mini", 1024, 1024, "high", 25501),
|
||||
("gpt-5.1", 1024, 1024, "auto", 630),
|
||||
("gpt-5", 1024, 1024, "low", 70),
|
||||
("o1", 1024, 1024, "high", 675),
|
||||
("o1-pro", 1024, 1024, "high", 675),
|
||||
("o3", 1024, 1024, "low", 75),
|
||||
("openai/gpt-4o-2024-08-06", 2048, 4096, "auto", 1105),
|
||||
("azure/gpt-4.1", 1024, 1024, "low", 85),
|
||||
("openrouter/openai/gpt-4o-mini", 16, 16, "low", 2833),
|
||||
("gpt-6-astra", 1024, 1024, "high", 1229),
|
||||
("gpt-6-astra", 2048, 2048, "high", 3000),
|
||||
("gpt-6-astra", 4096, 512, "high", 2458),
|
||||
("gpt-6-astra", 2048, 2048, "auto", 4916),
|
||||
("gpt-6-astra", 2048, 2048, "low", 308),
|
||||
("gpt-5.6-sol", 4096, 512, "high", 615),
|
||||
("gpt-5.6-terra", 2048, 2048, "original", 4916),
|
||||
("gpt-5.6-luna", 2048, 2048, "auto", 4916),
|
||||
("gpt-5.5", 6400, 6400, "original", 12000),
|
||||
("gpt-5.4", 2048, 2048, "low", 4916),
|
||||
("gpt-5.4-mini", 2048, 2048, "auto", 3000),
|
||||
("gpt-5.4-nano", 6400, 6400, "original", 12000),
|
||||
("gpt-5.2", 4096, 512, "auto", 615),
|
||||
("gpt-4.1-mini-2025-04-14", 1024, 1024, "low", 1659),
|
||||
("gpt-6-astra", 16, 16, "original", 2),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_uses_model_resolution_and_detail(
|
||||
model: str, width: int, height: int, detail: str, expected: int
|
||||
) -> None:
|
||||
assert estimate_image_tokens(model, _png_data_url(width, height), detail) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "data", "detail"),
|
||||
[
|
||||
("unregistered-model", _png_data_url(1024, 1024), "high"),
|
||||
("gpt-4.1-unrecognized", _png_data_url(1024, 1024), "high"),
|
||||
("other/gpt-4o", _png_data_url(1024, 1024), "high"),
|
||||
("gpt-4.1-nano", _png_data_url(1024, 1024), "high"),
|
||||
("gpt-4o", _png_data_url(1024, 1024), "original"),
|
||||
("gpt-5.2", _png_data_url(1024, 1024), "original"),
|
||||
("gpt-6-astra", _png_data_url(1024, 1024), "invalid"),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_falls_back_when_policy_is_unavailable(model: str, data: str, detail: str) -> None:
|
||||
assert estimate_image_tokens(model, data, detail) == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
|
||||
|
||||
|
||||
# Unavailable dimensions use documented limits, verified 2026-10-26:
|
||||
# https://developers.openai.com/api/docs/guides/images-vision (30,000 accepted patches before the multiplier)
|
||||
@pytest.mark.parametrize(
|
||||
("model", "detail", "expected", "data"),
|
||||
[
|
||||
("gpt-4o", "high", high_detail_image_token_upper_bound(), "https://image.invalid/photo.png"),
|
||||
("gpt-4o-mini", "auto", high_detail_image_token_upper_bound(2833, 5667), "https://image.invalid/photo.png"),
|
||||
("gpt-6-astra", "original", 36_000, "https://image.invalid/photo.png"),
|
||||
("gpt-6-astra", "high", 3000, "https://image.invalid/photo.png"),
|
||||
("gpt-6-astra", "low", 308, "https://image.invalid/photo.png"),
|
||||
("gpt-4o-mini", "low", 2833, "https://image.invalid/photo.png"),
|
||||
("gpt-6-astra", "high", 3000, "data:image/png;base64,invalid!"),
|
||||
("gpt-6-astra", "high", 3000, "data:application/octet-stream;base64,bm90LWFuLWltYWdl"),
|
||||
("gpt-6-astra", "high", 3000, _png_data_url(0, 1024)),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_uses_a_bounded_fallback_without_fetching_media(
|
||||
model: str, detail: str, expected: int, data: str,
|
||||
) -> None:
|
||||
with patch("httpx.Client.send") as send:
|
||||
estimate: Final = estimate_image_tokens(model, data, detail)
|
||||
send.assert_not_called()
|
||||
assert estimate == expected
|
||||
assert estimate >= max(
|
||||
estimate_image_tokens(model, _png_data_url(width, height), detail)
|
||||
for width, height in ((16, 16), (1024, 1024), (4096, 4096), (768, 2000))
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mime_type", ["image/png", "application/octet-stream"])
|
||||
def test_image_estimate_bounds_decoding_independent_of_payload_size(mime_type: str) -> None:
|
||||
header: Final = base64.b64decode(_png_data_url(1024, 1024).split(",", 1)[1])
|
||||
data: Final = f"data:{mime_type};base64," + base64.b64encode(header + b"\0" * 4_000_000).decode()
|
||||
with patch("base64.b64decode", wraps=base64.b64decode) as decode_header:
|
||||
estimate: Final = estimate_image_tokens(("gpt-6-astra", "azure/opaque"), data, "high", fallback_model="gpt-4o-mini")
|
||||
assert estimate == estimate_image_tokens("gpt-4o-mini", _png_data_url(1024, 1024), "high")
|
||||
assert decode_header.call_count == 1
|
||||
assert len(decode_header.call_args.args[0]) <= 65_536
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "canonical"),
|
||||
[
|
||||
("openai/gpt-4o-mini-2026-10-01", "gpt-4o-mini"),
|
||||
("azure/gpt-6-astra", "gpt-6-astra"),
|
||||
("azure/openai/gpt-4o-mini", "gpt-4o-mini"),
|
||||
("azure_ai/gpt-6-astra", "gpt-6-astra"),
|
||||
("azure_ai/openai/gpt-4o-mini", "gpt-4o-mini"),
|
||||
("openrouter/openai/gpt-6-astra", "gpt-6-astra"),
|
||||
("azure/responses/gpt-4o-mini", "gpt-4o-mini"),
|
||||
("azure/o_series/gpt-6-astra", "gpt-6-astra"),
|
||||
("azure/gpt-41", "gpt-4.1"),
|
||||
("azure/openai/gpt-41-mini", "gpt-4.1-mini"),
|
||||
("other/openai/gpt-4o", "unknown"),
|
||||
("azure/opaque/gpt-4o", "unknown"),
|
||||
("openai/azure/gpt-4o", "unknown"),
|
||||
("azure/openai/openai/gpt-4o", "unknown"),
|
||||
("openrouter/other/gpt-4o", "unknown"),
|
||||
("azure/gpt-4o-custom", "unknown"),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_resolves_only_supported_model_identities(model: str, canonical: str) -> None:
|
||||
data: Final = _png_data_url(1024, 1024)
|
||||
assert estimate_image_tokens(model, data, "high") == estimate_image_tokens(canonical, data, "high")
|
||||
|
||||
|
||||
def test_image_estimate_recognizes_cataloged_azure_names_without_caching_unknowns() -> None:
|
||||
model: Final = "azure/image-policy-test-region/gpt-4o-mini"
|
||||
data: Final = _png_data_url(1024, 1024)
|
||||
assert estimate_image_tokens(model, data, "high") == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
|
||||
|
||||
litellm.register_model(
|
||||
{model: {"litellm_provider": "azure", "mode": "chat", "input_cost_per_token": 0, "output_cost_per_token": 0}}
|
||||
)
|
||||
|
||||
assert estimate_image_tokens(model, data, "high") == estimate_image_tokens("gpt-4o-mini", data, "high")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("models", "fallback", "expected_model"),
|
||||
[
|
||||
(("azure/deployment-prod",), "gpt-4o", "gpt-4o"),
|
||||
(("openai/gpt-4o",), "gpt-4o-mini", "gpt-4o"),
|
||||
(("azure/deployment-prod", "openai/gpt-4o"), "gpt-4o-mini", "gpt-4o-mini"),
|
||||
(("openai/gpt-4o", "openai/gpt-4o-mini"), None, "gpt-4o-mini"),
|
||||
((), "gpt-6-astra", "gpt-6-astra"),
|
||||
(("other/gpt-4o-mini",), "unregistered-model", "unregistered-model"),
|
||||
(("azure/openai/gpt-4o",), "gpt-4o-mini", "gpt-4o"),
|
||||
(("azure/openai/deployment-prod",), "azure_ai/openai/gpt-4o-mini", "gpt-4o-mini"),
|
||||
(("azure/openai/gpt-4o", "azure/openai/deployment-prod"), "gpt-4o-mini", "gpt-4o-mini"),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_prefers_each_backend_policy_over_the_requested_model(
|
||||
models: tuple[str, ...], fallback: str | None, expected_model: str
|
||||
) -> None:
|
||||
data: Final = _png_data_url(1024, 1024)
|
||||
assert estimate_image_tokens(models, data, "high", fallback_model=fallback) == estimate_image_tokens(
|
||||
expected_model, data, "high"
|
||||
)
|
||||
|
||||
|
||||
def test_image_estimate_falls_back_when_dimensions_are_beyond_header_budget() -> None:
|
||||
jpeg: Final = (
|
||||
b"\xff\xd8\xff\xe1\xff\xff"
|
||||
+ b"\0" * 65533
|
||||
+ b"\xff\xc0\x00\x0b\x08"
|
||||
+ (512).to_bytes(2, "big")
|
||||
+ (1024).to_bytes(2, "big")
|
||||
)
|
||||
data: Final = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
|
||||
assert estimate_image_tokens("gpt-4o", data, "high") == high_detail_image_token_upper_bound()
|
||||
|
||||
|
||||
HUB_SETUP_SCRIPT: Final = """
|
||||
import json
|
||||
import sys
|
||||
|
|
|
|||
|
|
@ -15,19 +15,29 @@ Redis.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import wave
|
||||
from functools import partial
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, Final
|
||||
from typing import Any, Dict, Final, Literal
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm import Router, token_counter
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
|
||||
from litellm.litellm_core_utils.audio_utils.utils import AUDIO_BYTES_PER_TOKEN
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
PROJECT_ITPM_DESCRIPTOR_KEY,
|
||||
PROJECT_OTPM_DESCRIPTOR_KEY,
|
||||
RateLimitedModel,
|
||||
_AUDIO_BYTES_PER_TOKEN,
|
||||
_resolve_media_models,
|
||||
PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler,
|
||||
)
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
|
|
@ -37,6 +47,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
|||
get_request_stash,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache, hash_token
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.llms.openai import (
|
||||
InputTokensDetails,
|
||||
ResponseAPIUsage,
|
||||
|
|
@ -53,6 +64,397 @@ def rate_limiter():
|
|||
return handler, cache
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_cap,configured_output", [(None, None), (None, 32), (0, 32), (16, None)])
|
||||
def test_media_only_input_keeps_the_existing_output_budget(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], output_cap: int | None, configured_output: int | None,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
data: Final = {
|
||||
"messages": [{"role": "user", "content": [{"type": "image", "source": {"type": "file", "file_id": "opaque"}}]}],
|
||||
"n": 2, **({"max_tokens": output_cap} if output_cap is not None else {}),
|
||||
}
|
||||
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
|
||||
data, call_type="acompletion", configured_output_tokens=configured_output,
|
||||
)
|
||||
expected_output: Final = output_cap if output_cap is not None else configured_output or handler.no_max_tokens_output_floor(None)
|
||||
assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT
|
||||
assert output_tokens == expected_output * 2
|
||||
|
||||
|
||||
def _audio_message(seconds: int, sample_rate: int, channels: int) -> dict[str, object]:
|
||||
output: Final = io.BytesIO()
|
||||
with wave.open(output, "wb") as wav:
|
||||
wav.setnchannels(channels)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(b"\x00\x00" * seconds * sample_rate * channels)
|
||||
return {"role": "user", "content": [{
|
||||
"type": "input_audio", "input_audio": {
|
||||
"data": base64.b64encode(output.getvalue()).decode(), "format": "wav",
|
||||
},
|
||||
}]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("detail,admitted", [("low", True), ("high", False)])
|
||||
async def test_model_specific_images_resolve_routes_before_admission(
|
||||
detail: str, admitted: bool, equivalent_image_urls: tuple[str, str],
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[{
|
||||
"model_name": "vision", "litellm_params": {
|
||||
"model": "azure/custom-deployment", "base_model": "gpt-4o-mini",
|
||||
"api_key": "test", "api_base": "https://example.invalid", "api_version": "2024-12-01-preview",
|
||||
}, "model_info": {"base_model": "gpt-6-astra"},
|
||||
}], model_group_alias={"alias": "vision"},
|
||||
)
|
||||
cache: Final = DualCache()
|
||||
handler: Final = RateLimitHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), media_model_resolver=partial(_resolve_media_models, router),
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-image-policy"), tpm_limit=500)
|
||||
data: Final = {"model": "alias", "messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": equivalent_image_urls[1], "detail": detail}},
|
||||
]}], "max_tokens": 16}
|
||||
if not admitted:
|
||||
with pytest.raises(ProxyRateLimitError):
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
return
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
assert 250 < stash.reserved_tokens < 500
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def equivalent_image_urls() -> tuple[str, str]:
|
||||
from PIL import Image
|
||||
|
||||
def encoded_image(compression: int) -> str:
|
||||
output: Final = io.BytesIO()
|
||||
Image.new("RGB", (1024, 1024)).save(output, format="PNG", compress_level=compression)
|
||||
return "data:image/png;base64," + base64.b64encode(output.getvalue()).decode()
|
||||
|
||||
return encoded_image(0), encoded_image(9)
|
||||
|
||||
|
||||
def _structured_image_request(
|
||||
image_url: str, representation: str, detail: str = "high"
|
||||
) -> tuple[str, dict[str, object]]:
|
||||
image: Final = {"type": "image_url", "image_url": {"url": image_url, "detail": detail}}
|
||||
text: Final = {"type": "text", "text": "Describe this image"}
|
||||
base: Final = {"model": "gpt-6-astra"}
|
||||
if representation == "chat":
|
||||
return "acompletion", {**base, "messages": [{"role": "user", "content": [text, image]}], "max_tokens": 16}
|
||||
if representation == "chat_input_image":
|
||||
input_image: Final = {**image, "type": "input_image"}
|
||||
return "acompletion", {**base, "messages": [{"role": "user", "content": [text, input_image]}], "max_tokens": 16}
|
||||
if representation == "anthropic_tool":
|
||||
source_image: Final = {"type": "image", "source": {
|
||||
"type": "base64", "media_type": "image/png", "data": image_url.partition(",")[2],
|
||||
}}
|
||||
return "anthropic_messages", {**base, "messages": [{"role": "user", "content": [{
|
||||
"type": "tool_result", "tool_use_id": "call_image", "content": [text, source_image],
|
||||
}]}], "max_tokens": 16}
|
||||
if representation == "document":
|
||||
return "anthropic_messages", {**base, "messages": [{"role": "user", "content": [{
|
||||
"type": "document", "source": {"type": "content", "content": [text, image]},
|
||||
}]}], "max_tokens": 16}
|
||||
response_content: Final = [
|
||||
{"type": "input_text", "text": "Describe this image"},
|
||||
{"type": "input_image", "image_url": image_url, "detail": detail},
|
||||
]
|
||||
response_input: Final = (
|
||||
[{"type": "custom_tool_call_output", "call_id": "call_image", "output": response_content}]
|
||||
if representation == "custom_output"
|
||||
else [{"role": "user", "content": response_content}]
|
||||
)
|
||||
return "aresponses", {**base, "input": response_input, "max_output_tokens": 16}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("representation", ["chat", "chat_input_image", "anthropic_tool", "document", "responses", "custom_output"])
|
||||
def test_structured_media_estimates_ignore_transport_size_in_every_quota(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], equivalent_image_urls: tuple[str, str], representation: str,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
uncompressed, compressed = equivalent_image_urls
|
||||
assert len(uncompressed) > 100 * len(compressed)
|
||||
call_type, first = _structured_image_request(uncompressed, representation)
|
||||
_, second = _structured_image_request(compressed, representation)
|
||||
original: Final = json.dumps(first)
|
||||
first_input: Final = handler._estimate_request_input_tokens(first, call_type=call_type, precise=True)
|
||||
second_input: Final = handler._estimate_request_input_tokens(second, call_type=call_type, precise=True)
|
||||
assert first_input == second_input
|
||||
assert 500 < first_input.cheap_tokens < 5000
|
||||
assert first_input.precise_tokens is not None and 500 < first_input.precise_tokens < 5000
|
||||
assert handler._estimate_input_and_output_tokens(first, call_type=call_type) == (
|
||||
first_input.cheap_tokens, 16,
|
||||
)
|
||||
assert json.dumps(first) == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_kind", ["function_call", "custom_tool_call"])
|
||||
def test_responses_tool_arguments_grow_reservations(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache],
|
||||
equivalent_image_urls: tuple[str, str],
|
||||
tool_kind: str,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
output_request: Final = _structured_image_request(equivalent_image_urls[1], "custom_output")[1]
|
||||
tool_output: Final = TypeAdapter(list[dict[str, object]]).validate_python(output_request["input"])[0]
|
||||
requests: Final = tuple(
|
||||
{
|
||||
**output_request,
|
||||
"input": [
|
||||
{
|
||||
"type": tool_kind, "call_id": "call_image", "name": "inspect",
|
||||
**(
|
||||
{"arguments": json.dumps({"query": "a" * size})}
|
||||
if tool_kind == "function_call" else {"input": "a" * size}
|
||||
),
|
||||
},
|
||||
{
|
||||
**tool_output,
|
||||
"type": tool_kind + "_output",
|
||||
},
|
||||
],
|
||||
}
|
||||
for size in (8, 8008)
|
||||
)
|
||||
estimates: Final = tuple(
|
||||
handler._estimate_request_input_tokens(data, call_type="aresponses", precise=True) for data in requests
|
||||
)
|
||||
assert estimates[1].cheap_tokens - estimates[0].cheap_tokens == 2000
|
||||
|
||||
chat_estimates: Final = tuple(
|
||||
handler._estimate_request_input_tokens(
|
||||
{"model": "gpt-6-astra", "messages": handler._responses_input_to_chat_messages(data)},
|
||||
call_type="acompletion",
|
||||
).cheap_tokens
|
||||
for data in requests
|
||||
)
|
||||
assert chat_estimates[0] == chat_estimates[1]
|
||||
with_tools: Final = {
|
||||
**requests[1], "tools": [{"type": "function", "name": "inspect", "description": "d" * 12000}]
|
||||
}
|
||||
assert (
|
||||
handler._estimate_request_input_tokens(with_tools, call_type="aresponses").cheap_tokens
|
||||
== estimates[1].cheap_tokens
|
||||
)
|
||||
|
||||
unsupported_content: Final = {
|
||||
**requests[1], "input": [
|
||||
*TypeAdapter(list[dict[str, object]]).validate_python(requests[1]["input"]),
|
||||
{"role": "user", "content": [
|
||||
{"type": "guarded_text", "text": ""},
|
||||
{"type": "input_file", "filename": "opaque.pdf", "file_data": "data:application/pdf;base64,eA=="},
|
||||
]},
|
||||
],
|
||||
}
|
||||
with pytest.raises(ValueError, match="guarded_text"):
|
||||
token_counter(
|
||||
model="gpt-6-astra", messages=handler._responses_input_to_chat_messages(unsupported_content),
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
fallback: Final = handler._estimate_request_input_tokens(unsupported_content, call_type="aresponses", precise=True)
|
||||
assert fallback.precise_tokens == fallback.cheap_tokens == estimates[1].cheap_tokens + DEFAULT_IMAGE_TOKEN_COUNT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("representation", ["chat", "responses", "anthropic_tool", "document_content"])
|
||||
def test_opaque_documents_keep_a_cheap_reservation_and_existing_precise_count(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], representation: str,
|
||||
) -> None:
|
||||
from pypdf import PdfWriter
|
||||
|
||||
handler, _ = rate_limiter
|
||||
writer: Final = PdfWriter()
|
||||
writer.add_blank_page(width=612, height=792)
|
||||
writer.add_blank_page(width=612, height=792)
|
||||
output: Final = io.BytesIO()
|
||||
writer.write(output)
|
||||
encoded: Final = base64.b64encode(output.getvalue()).decode()
|
||||
file: Final = {"type": "file", "file": {"filename": "document.pdf", "file_data": "data:application/pdf;base64," + encoded}}
|
||||
document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": encoded}}
|
||||
content: Final = (
|
||||
[{"type": "tool_result", "tool_use_id": "read", "content": [document, document]}]
|
||||
if representation == "anthropic_tool"
|
||||
else [{"type": "document", "source": {"type": "content", "content": [file, file]}}]
|
||||
if representation == "document_content"
|
||||
else [file, file]
|
||||
)
|
||||
request: Final = (
|
||||
{"model": "gpt-6-astra", "input": [{"role": "user", "content": [
|
||||
{"type": "input_file", **file["file"]}, {"type": "input_file", **file["file"]},
|
||||
]}], "max_output_tokens": 16}
|
||||
if representation == "responses"
|
||||
else {"model": "gpt-6-astra", "messages": [{"role": "user", "content": content}], "max_tokens": 16}
|
||||
)
|
||||
call_type: Final = "aresponses" if representation == "responses" else "acompletion"
|
||||
original: Final = json.dumps(request)
|
||||
estimate: Final = handler._estimate_request_input_tokens(request, call_type=call_type, precise=True)
|
||||
assert estimate.cheap_tokens == 2 * DEFAULT_IMAGE_TOKEN_COUNT
|
||||
assert handler._estimate_input_and_output_tokens(request, call_type=call_type) == (estimate.cheap_tokens, 16)
|
||||
messages: Final = handler._responses_input_to_chat_messages(request) if representation == "responses" else request["messages"]
|
||||
assert estimate.precise_tokens == token_counter(
|
||||
model="gpt-6-astra", messages=messages, use_default_image_token_count=True,
|
||||
)
|
||||
assert estimate.precise_tokens > estimate.cheap_tokens
|
||||
assert json.dumps(request) == original
|
||||
for _ in range(18):
|
||||
writer.add_blank_page(width=612, height=792)
|
||||
large_output: Final = io.BytesIO()
|
||||
writer.write(large_output)
|
||||
large_encoded: Final = base64.b64encode(large_output.getvalue()).decode()
|
||||
large_request: Final = TypeAdapter(dict[str, object]).validate_json(original.replace(encoded, large_encoded))
|
||||
large_estimate: Final = handler._estimate_request_input_tokens(large_request, call_type=call_type)
|
||||
assert large_estimate.cheap_tokens > estimate.cheap_tokens
|
||||
assert large_estimate.cheap_tokens >= len(large_encoded) // 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("quota", ["combined", "project", "ptu"])
|
||||
@pytest.mark.parametrize("detail,limit,admitted", [("low", 500, True), ("high", 500, False), ("high", 2000, True)])
|
||||
async def test_image_detail_controls_each_admission_path(
|
||||
equivalent_image_urls: tuple[str, str], quota: str, detail: str, limit: int, admitted: bool,
|
||||
) -> None:
|
||||
def ceiling(_team: str, model: str) -> PTUTeamCeiling | None:
|
||||
return PTUTeamCeiling(model_group=model, tpm_limit=limit, output_to_input_ratio=4.0, cached_input_ratio=0.0) if quota == "ptu" else None
|
||||
|
||||
cache: Final = DualCache()
|
||||
handler: Final = RateLimitHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=ceiling,
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-detail-quota"), team_id="media-team",
|
||||
tpm_limit=limit if quota == "combined" else None, project_id="media-project",
|
||||
project_metadata={"model_itpm_limit": {"gpt-6-astra": limit}} if quota == "project" else {},
|
||||
)
|
||||
call_type, data = _structured_image_request(equivalent_image_urls[1], "chat", detail)
|
||||
if admitted:
|
||||
await handler.async_pre_call_hook(auth, cache, data, call_type)
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
assert max(stash.reserved_tokens, stash.itpm_reserved_tokens, stash.ptu_reserved_tokens) > 250
|
||||
return
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await handler.async_pre_call_hook(auth, cache, data, call_type)
|
||||
assert error.value.status_code == (400 if quota == "ptu" else 429)
|
||||
|
||||
|
||||
def test_precise_tokenizer_fallback_reuses_nested_media_and_visible_text_once(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], equivalent_image_urls: tuple[str, str],
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
call_type, request = _structured_image_request(equivalent_image_urls[1], "anthropic_tool")
|
||||
data: Final = {**request, "messages": [
|
||||
*TypeAdapter(list[object]).validate_python(request["messages"]),
|
||||
_audio_message(31, 48000, 2),
|
||||
{"role": "user", "content": [{"type": "unsupported_counter_block"}]},
|
||||
]}
|
||||
measured: Final = handler._estimate_request_input_tokens(data, call_type=call_type, precise=True)
|
||||
image_only: Final = handler._estimate_request_input_tokens(request, call_type=call_type)
|
||||
assert measured.precise_tokens == measured.cheap_tokens == image_only.cheap_tokens + 310
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("outcome", ["success", "failure", "partial", "project_rejection"])
|
||||
async def test_mixed_media_reservations_settle_each_bucket_in_its_own_units(
|
||||
equivalent_image_urls: tuple[str, str], outcome: Literal["success", "failure", "partial", "project_rejection"],
|
||||
) -> None:
|
||||
now: Final = datetime(2026, 1, 2, 3, 4, 5)
|
||||
|
||||
def ceiling(_team: str, model: str) -> PTUTeamCeiling:
|
||||
return PTUTeamCeiling(model_group=model, tpm_limit=10000, output_to_input_ratio=4.0, cached_input_ratio=0.0)
|
||||
|
||||
cache: Final = DualCache()
|
||||
handler: Final = RateLimitHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), time_provider=lambda: now, ptu_team_ceiling_resolver=ceiling,
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-media-settlement"), tpm_limit=10000, team_id="media-team", project_id="media-project",
|
||||
project_metadata={
|
||||
"model_itpm_limit": {"gpt-6-astra": 500 if outcome == "project_rejection" else 2000},
|
||||
"model_otpm_limit": {"gpt-6-astra": 10000},
|
||||
},
|
||||
)
|
||||
_, image_request = _structured_image_request(equivalent_image_urls[1], "chat")
|
||||
data: Final = {**image_request, "messages": [*TypeAdapter(list[object]).validate_python(image_request["messages"]), _audio_message(31, 24000, 1)]}
|
||||
scopes: Final = (
|
||||
("api_key", auth.api_key),
|
||||
("model_per_team_ptu", "media-team:gpt-6-astra"),
|
||||
(PROJECT_ITPM_DESCRIPTOR_KEY, "media-project:gpt-6-astra"),
|
||||
(PROJECT_OTPM_DESCRIPTOR_KEY, "media-project:gpt-6-astra"),
|
||||
)
|
||||
if outcome == "project_rejection":
|
||||
with pytest.raises(ProxyRateLimitError):
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
else:
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
assert stash.reserved_tokens > 1500
|
||||
assert stash.itpm_reserved_tokens > 1500
|
||||
assert stash.ptu_reserved_tokens == stash.reserved_tokens + 3 * stash.otpm_reserved_tokens
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=40, completion_tokens=10, total_tokens=50,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=15),
|
||||
)
|
||||
kwargs: Final = {
|
||||
"litellm_call_id": data.get("litellm_call_id"),
|
||||
"model": "gpt-6-astra",
|
||||
"standard_logging_object": {"metadata": {
|
||||
"user_api_key_hash": auth.api_key, "user_api_key_team_id": auth.team_id,
|
||||
"user_api_key_project_id": auth.project_id,
|
||||
}},
|
||||
**({"combined_usage_object": usage} if outcome == "partial" else {}),
|
||||
}
|
||||
if outcome == "success":
|
||||
response: Final = ModelResponse(model="gpt-6-astra", usage=usage, choices=[])
|
||||
await handler.async_log_success_event(kwargs, response, now, now)
|
||||
else:
|
||||
await handler.async_log_failure_event(kwargs, None, now, now)
|
||||
await handler.async_log_failure_event(kwargs, None, now, now)
|
||||
expected: Final = (35, 65, 25, 10) if outcome in ("success", "partial") else (0, 0, 0, 0)
|
||||
for (scope, value), amount in zip(scopes, expected):
|
||||
key: Final = handler.create_rate_limit_keys(scope, value or "", "tokens")
|
||||
assert int(await cache.async_get_cache(key=key, local_only=True) or 0) == amount
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("representation", ["chat", "inlineData"])
|
||||
@pytest.mark.parametrize("seconds,admitted", [(1, True), (31, False)])
|
||||
async def test_native_and_chat_audio_share_duration_based_quota_admission(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], representation: str, seconds: int, admitted: bool,
|
||||
) -> None:
|
||||
handler, cache = rate_limiter
|
||||
model: Final = "gemini/gemini-3.8-flash"
|
||||
message: Final = _audio_message(seconds, 24000, 1)
|
||||
blocks: Final = TypeAdapter(list[dict[str, object]]).validate_python(message["content"])
|
||||
audio: Final = TypeAdapter(dict[str, str]).validate_python(blocks[0]["input_audio"])
|
||||
native: Final = representation != "chat"
|
||||
data: Final = (
|
||||
{"model": model, "contents": [{"role": "user", "parts": [{
|
||||
"inlineData": {"mimeType": "audio/wav", "data": audio["data"]},
|
||||
}]}], "config": {"maxOutputTokens": 16}}
|
||||
if native
|
||||
else {"model": model, "messages": [message], "max_tokens": 16}
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-native-duration"), tpm_limit=100, project_id="audio-project",
|
||||
project_metadata={"model_itpm_limit": {model: 100}},
|
||||
)
|
||||
call_type: Final = "agenerate_content" if native else "acompletion"
|
||||
if admitted:
|
||||
await handler.async_pre_call_hook(auth, cache, data, call_type)
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
assert 26 < stash.reserved_tokens < 100
|
||||
assert 10 < stash.itpm_reserved_tokens < 100
|
||||
return
|
||||
with pytest.raises(ProxyRateLimitError):
|
||||
await handler.async_pre_call_hook(auth, cache, data, call_type)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_request_stash():
|
||||
token = _request_stash.set(None)
|
||||
|
|
@ -2274,7 +2676,7 @@ def test_audio_token_estimate_scales_with_payload_size():
|
|||
to exhaust ITPM quota while reserving almost nothing.
|
||||
|
||||
The estimate must now grow proportionally with the base64 payload size
|
||||
(len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN), floored at
|
||||
(len(b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN), floored at
|
||||
DEFAULT_AUDIO_TOKEN_ESTIMATE so reference-only blocks and genuinely
|
||||
short clips still get a non-trivial reservation.
|
||||
|
||||
|
|
@ -2312,7 +2714,7 @@ def test_audio_token_estimate_scales_with_payload_size():
|
|||
f"Large payload ({large_estimate}) must reserve more than small payload "
|
||||
f"({small_estimate}); flat-rate bug is back"
|
||||
)
|
||||
assert very_large_estimate == len(very_large_b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN
|
||||
assert very_large_estimate == len(very_large_b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN
|
||||
assert very_large_estimate > 6_000
|
||||
assert no_data_estimate >= 300, (
|
||||
f"Reference-only block (no data) must use the DEFAULT_AUDIO_TOKEN_ESTIMATE floor; got {no_data_estimate}"
|
||||
|
|
@ -2792,26 +3194,6 @@ async def test_reserve_io_tokens_itpm_only_no_otpm(rate_limiter):
|
|||
assert otpm_reserved == 0
|
||||
|
||||
|
||||
def test_strip_audio_content_blocks_passthrough_non_list_messages():
|
||||
"""Non-list input is returned unchanged (early return on line 2605)."""
|
||||
result = RateLimitHandler._strip_audio_content_blocks("not a list")
|
||||
assert result == "not a list"
|
||||
|
||||
|
||||
def test_strip_audio_content_blocks_passthrough_non_dict_message():
|
||||
"""Non-dict entries in the message list are appended unchanged."""
|
||||
messages = ["plain string message"]
|
||||
result = RateLimitHandler._strip_audio_content_blocks(messages)
|
||||
assert result == ["plain string message"]
|
||||
|
||||
|
||||
def test_strip_audio_content_blocks_passthrough_non_list_content():
|
||||
"""Messages with non-list content (e.g. plain string) pass through unchanged."""
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
result = RateLimitHandler._strip_audio_content_blocks(messages)
|
||||
assert result == [{"role": "user", "content": "hello"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter):
|
||||
"""
|
||||
|
|
@ -3519,6 +3901,28 @@ def test_split_token_estimate_selects_endpoint_input(rate_limiter, call_type, da
|
|||
assert estimated_output == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call_type,data,cheap_input,precise_input",
|
||||
[
|
||||
("embedding", {"input": [1, 2, 3]}, 1, 3),
|
||||
("embedding", {"input": [[1, 2], [3]]}, 2, 3),
|
||||
("rerank", {"query": "query text", "documents": ["first document", "second document"]}, 0, None),
|
||||
],
|
||||
)
|
||||
def test_nonmedia_input_keeps_distinct_cheap_and_precise_reservations(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], call_type: str, data: dict[str, object],
|
||||
cheap_input: int, precise_input: int | None,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
estimate: Final = handler._estimate_request_input_tokens(data, "gpt-6-astra", call_type, precise=True)
|
||||
assert handler._estimate_input_and_output_tokens(
|
||||
data, min_configured_tpm_limit=100, call_type=call_type, input_estimate=estimate,
|
||||
) == (cheap_input, 0)
|
||||
assert estimate.precise_tokens is not None and estimate.precise_tokens > cheap_input
|
||||
if precise_input is not None:
|
||||
assert estimate.precise_tokens == precise_input
|
||||
|
||||
|
||||
def test_split_quota_multimodal_guards_handle_non_mapping_inputs(rate_limiter):
|
||||
handler, _cache = rate_limiter
|
||||
|
||||
|
|
|
|||
|
|
@ -20261,8 +20261,9 @@ def _resolution_router() -> Router:
|
|||
],
|
||||
ids=["exact-name", "model-group-alias", "every-member-of-a-pool", "wildcard-expands", "resolves-to-nothing"],
|
||||
)
|
||||
@pytest.mark.parametrize("use_base_model", (False, True))
|
||||
def test_resolved_litellm_models_answers_through_every_channel_a_request_uses(
|
||||
model_name: str, expected: tuple[str, ...]
|
||||
model_name: str, expected: tuple[str, ...], use_base_model: bool
|
||||
) -> None:
|
||||
"""A caller comparing two names by what serves them needs each channel the request path
|
||||
composes, since the deployment name an admin picked carries no information on its own.
|
||||
|
|
@ -20270,7 +20271,7 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses(
|
|||
`resolves-to-nothing` is the contract that keeps the fallback out of here: an empty
|
||||
result is not "the call fails", so what to do about it stays each caller's policy.
|
||||
"""
|
||||
assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected)
|
||||
assert set(_resolution_router().resolved_litellm_models(model_name, use_base_model=use_base_model)) == set(expected)
|
||||
|
||||
|
||||
class TestTierParamsTheTargetAccepts:
|
||||
|
|
@ -30149,3 +30150,67 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items():
|
|||
assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True
|
||||
assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False
|
||||
assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("info_base,params_base", [(None, None), (None, "gpt-5.1"), ("gpt-6-astra", "gpt-5.1")])
|
||||
def test_resolved_litellm_models_can_use_configured_base_model(
|
||||
info_base: str | None, params_base: str | None,
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[{
|
||||
"model_name": "deployment", "litellm_params": {
|
||||
"model": "azure/custom-name", "api_key": "test", "api_base": "https://example.invalid",
|
||||
"api_version": "2024-12-01-preview", **({"base_model": params_base} if params_base else {}),
|
||||
}, "model_info": {"base_model": info_base}, "rpm": 10.5,
|
||||
}], model_group_alias={"alias": "deployment"},
|
||||
)
|
||||
assert router.resolved_litellm_models("alias") == ("azure/custom-name",)
|
||||
assert router.resolved_litellm_models("alias", use_base_model=True) == (
|
||||
info_base or params_base or "azure/custom-name",
|
||||
)
|
||||
|
||||
|
||||
def test_resolved_model_identity_does_not_revalidate_stored_quotas() -> None:
|
||||
router: Final = Router(
|
||||
model_list=[{
|
||||
"model_name": "vision/*",
|
||||
"litellm_params": {
|
||||
"model": "azure/vision-deployment", "api_key": "test", "api_base": "https://example.invalid",
|
||||
"api_version": "2024-12-01-preview", "base_model": "gpt-5.1",
|
||||
},
|
||||
"model_info": {"base_model": "gpt-6-astra"},
|
||||
"tpm": 10.5,
|
||||
}],
|
||||
ignore_invalid_deployments=False,
|
||||
)
|
||||
|
||||
assert router.resolved_litellm_models("vision/image") == ("azure/vision-deployment",)
|
||||
assert router.resolved_litellm_models("vision/image", use_base_model=True) == ("gpt-6-astra",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("explicit_provider", (False, True))
|
||||
@pytest.mark.parametrize("profile_known", (False, True))
|
||||
def test_resolved_model_identity_uses_only_mapped_bedrock_profiles(
|
||||
monkeypatch: pytest.MonkeyPatch, explicit_provider: bool, profile_known: bool,
|
||||
) -> None:
|
||||
profile: Final = "us.anthropic.identity-test"
|
||||
model: Final = "converse/anthropic.identity-test"
|
||||
monkeypatch.setattr(litellm, "model_cost", {
|
||||
**litellm.model_cost,
|
||||
**({profile: {
|
||||
"litellm_provider": "bedrock", "mode": "chat",
|
||||
"input_cost_per_token": 0.01, "output_cost_per_token": 0.02,
|
||||
}} if profile_known else {}),
|
||||
})
|
||||
configured_model: Final = model if explicit_provider else f"bedrock/{model}"
|
||||
router: Final = Router(model_list=[{
|
||||
"model_name": "profile", "litellm_params": {
|
||||
"model": configured_model, "model_id": profile,
|
||||
**({"custom_llm_provider": "bedrock"} if explicit_provider else {}),
|
||||
},
|
||||
}])
|
||||
|
||||
assert router.resolved_litellm_models("profile") == (configured_model,)
|
||||
assert router.resolved_litellm_models("profile", use_base_model=True) == (
|
||||
f"bedrock/{profile}" if profile_known else configured_model,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue