mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
Merge ce931d1fb5 into dddbaa803d
This commit is contained in:
commit
7dce87d487
6 changed files with 624 additions and 39 deletions
|
|
@ -3,9 +3,11 @@
|
|||
import base64
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from itertools import accumulate
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
||||
import anyio
|
||||
|
|
@ -342,6 +344,117 @@ def calculate_img_tokens(
|
|||
return total_tokens
|
||||
|
||||
|
||||
_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
|
||||
_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
|
||||
{
|
||||
"gpt-4o": (85, 170),
|
||||
"gpt-4.1": (85, 170),
|
||||
"gpt-4o-mini": (2833, 5667),
|
||||
"gpt-5": (70, 140),
|
||||
"gpt-5.1": (70, 140),
|
||||
"o1": (75, 150),
|
||||
"o1-pro": (75, 150),
|
||||
"o3": (75, 150),
|
||||
}
|
||||
)
|
||||
_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
|
||||
{
|
||||
"gpt-6-astra": 1.2,
|
||||
"gpt-5.6-sol": 1.2,
|
||||
"gpt-5.6-terra": 1.2,
|
||||
"gpt-5.6-luna": 1.2,
|
||||
"gpt-5.5": 1.2,
|
||||
"gpt-5.4": 1.2,
|
||||
"gpt-5.4-mini": 1.2,
|
||||
"gpt-5.4-nano": 1.2,
|
||||
"gpt-5.2": 1.2,
|
||||
"gpt-4.1-mini": 1.62,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
|
||||
separator: Final = data.find(",", 0, 256)
|
||||
if not data.startswith("data:image/") or separator < 0 or not data[:separator].endswith(";base64"):
|
||||
return None
|
||||
encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
|
||||
try:
|
||||
header: Final = base64.b64decode(encoded_header, validate=True)
|
||||
image_type: Final = get_image_type(header)
|
||||
if image_type not in ("png", "gif", "jpeg", "webp"):
|
||||
return None
|
||||
if image_type == "webp" and header[12:16] not in (b"VP8X", b"VP8 ", b"VP8L"):
|
||||
return None
|
||||
width, height = get_image_dimensions(f"data:image/{image_type};base64,{encoded_header}")
|
||||
except (ValueError, TypeError, struct.error):
|
||||
return None
|
||||
return (width, height) if width > 0 and height > 0 else None
|
||||
|
||||
|
||||
def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
|
||||
if model in ("gpt-5.2", "gpt-4.1-mini"):
|
||||
return None if detail == "original" else (2048, 6144)
|
||||
original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
|
||||
effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
|
||||
if effective_detail == "low":
|
||||
return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
|
||||
if effective_detail == "high":
|
||||
return (65_535 if model == "gpt-6-astra" else 2048, 2500)
|
||||
return (
|
||||
(65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
|
||||
)
|
||||
|
||||
|
||||
def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
|
||||
dimension_scale: Final = min(1.0, max_dimension / max(width, height))
|
||||
fitted_width: Final = max(1, round(width * dimension_scale))
|
||||
fitted_height: Final = max(1, round(height * dimension_scale))
|
||||
patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
|
||||
if budget is None or patches <= budget:
|
||||
return patches
|
||||
shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
|
||||
width_patches: Final = fitted_width * shrink / 32
|
||||
height_patches: Final = fitted_height * shrink / 32
|
||||
aligned_shrink: Final = shrink * min(
|
||||
math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
|
||||
)
|
||||
resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
|
||||
resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
|
||||
return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
|
||||
|
||||
|
||||
def estimate_image_tokens(model: str, data: str, detail: str = "auto") -> int:
|
||||
"""Estimate documented image policies from at most 48 KiB of inline headers, without fetching media."""
|
||||
if detail not in ("low", "high", "auto", "original"):
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
provider_model: Final = re.sub(r"^(?:openai|azure|openrouter/openai)/", "", model)
|
||||
family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
|
||||
tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
|
||||
multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
|
||||
if (tile_costs is None and multiplier is None) or (tile_costs is not None and detail == "original"):
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
if tile_costs is not None and detail == "low" and data.startswith(("https://", "http://")):
|
||||
return tile_costs[0]
|
||||
dimensions: Final = _inline_image_dimensions(data)
|
||||
if dimensions is None:
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
width, height = dimensions
|
||||
if tile_costs is not None:
|
||||
base_tokens, tile_tokens = tile_costs
|
||||
if detail == "low":
|
||||
return base_tokens
|
||||
scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
|
||||
tiles: Final = math.ceil(max(1, math.floor(width * scale)) / 512) * math.ceil(
|
||||
max(1, math.floor(height * scale)) / 512
|
||||
)
|
||||
return base_tokens + tiles * tile_tokens
|
||||
limits: Final = _patch_image_limits(family, detail)
|
||||
if limits is None or multiplier is None:
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
max_dimension, patch_budget = limits
|
||||
return math.ceil(_image_patch_count(width, height, max_dimension, patch_budget) * multiplier)
|
||||
|
||||
|
||||
TokenCounterFunction = Callable[[str], int]
|
||||
"""
|
||||
Type for a function that counts tokens in a string.
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ This is currently in development and not yet ready for production.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set
|
||||
|
|
@ -42,14 +44,19 @@ from litellm.caching.redis_batch import (
|
|||
active_request_redis_batch,
|
||||
)
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.constants import (
|
||||
DEFAULT_IMAGE_TOKEN_COUNT,
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import normalized_tokens
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
|
|
@ -98,6 +105,7 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.router import Router
|
||||
from litellm.types.agents import AgentResponse
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
|
||||
|
|
@ -109,6 +117,7 @@ else:
|
|||
|
||||
|
||||
_REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object])
|
||||
_MESSAGE_CONTENT_OBJECTS: Final = TypeAdapter(list[Mapping[str, object]])
|
||||
_MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None)
|
||||
|
||||
|
||||
|
|
@ -135,6 +144,28 @@ class RateLimitedModel:
|
|||
return requested_limit if requested_limit is not None else limits.get(self.group)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MediaTokenEstimate:
|
||||
image_tokens: int = 0
|
||||
audio_tokens: int = 0
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return self.image_tokens + self.audio_tokens
|
||||
|
||||
|
||||
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
if router is None:
|
||||
return (model,)
|
||||
return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,)
|
||||
|
||||
|
||||
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return _resolve_media_models(llm_router, model, team_id)
|
||||
|
||||
|
||||
def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
|
|
@ -502,17 +533,8 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
|
|||
CallTypes.agenerate_content_stream.value,
|
||||
)
|
||||
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
|
||||
# litellm.token_counter has no per-type handling for "input_audio" content
|
||||
# blocks (unlike images, which use use_default_image_token_count) -- it
|
||||
# silently contributes 0 tokens for them. When the block carries a base64
|
||||
# payload, the estimate is derived from the decoded byte count; when the
|
||||
# block is a reference without a payload (or the payload is missing), this
|
||||
# flat per-block floor is used instead.
|
||||
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
|
||||
# Conservative bytes-per-token assumption for size-based audio estimation:
|
||||
# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing
|
||||
# the lowest reasonable bitrate means we never under-reserve for higher-
|
||||
# quality audio recorded at the same wall-clock duration.
|
||||
_AUDIO_TOKENS_PER_SECOND: Final = 10
|
||||
_AUDIO_BYTES_PER_TOKEN: Final = 1600
|
||||
# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from
|
||||
# "model_per_project" (the combined-TPM descriptor) so both can be enforced
|
||||
|
|
@ -867,11 +889,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
] = _resolve_ptu_team_ceiling_via_proxy_router,
|
||||
fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
|
||||
force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
|
||||
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
|
||||
):
|
||||
self.internal_usage_cache = internal_usage_cache
|
||||
self._time_provider = time_provider or datetime.now
|
||||
self._tag_rate_limit_resolver = tag_rate_limit_resolver
|
||||
self._model_group_resolver = model_group_resolver
|
||||
self._media_model_resolver = media_model_resolver
|
||||
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
|
||||
self._fail_closed_resolver = fail_closed_resolver
|
||||
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
|
||||
|
|
@ -1138,6 +1162,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit: int | None = None,
|
||||
call_type: str | None = None,
|
||||
configured_output_tokens: int | None = None,
|
||||
media_estimate: MediaTokenEstimate | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Estimate total tokens this request will consume so we can reserve them
|
||||
|
|
@ -1164,6 +1189,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
media_estimate=media_estimate,
|
||||
)
|
||||
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
|
||||
|
||||
|
|
@ -1184,6 +1210,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type: str | None,
|
||||
configured_output_tokens: int | None,
|
||||
raw_estimate: int,
|
||||
media_estimate: MediaTokenEstimate | None = None,
|
||||
) -> int:
|
||||
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
|
||||
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
|
||||
|
|
@ -1194,6 +1221,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
media_estimate=media_estimate,
|
||||
)
|
||||
normalized: Final = normalized_tokens(
|
||||
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
|
||||
|
|
@ -1206,6 +1234,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit: int | None = None,
|
||||
call_type: str | None = None,
|
||||
configured_output_tokens: int | None = None,
|
||||
media_estimate: MediaTokenEstimate | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Estimate input tokens and output (max_tokens) budget separately, so
|
||||
|
|
@ -1232,7 +1261,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if not isinstance(data, dict):
|
||||
return 0, 0
|
||||
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
|
||||
estimable_data: Final = translated_data if translated_data is not None else data
|
||||
estimable_data: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(
|
||||
translated_data if translated_data is not None else data
|
||||
)
|
||||
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
|
||||
(None, None, estimable_data.get("input"))
|
||||
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
|
||||
|
|
@ -1253,6 +1284,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
|
||||
)
|
||||
|
||||
model_value: Final = estimable_data.get("model")
|
||||
estimated_media_tokens: Final = (
|
||||
media_estimate
|
||||
if media_estimate is not None
|
||||
else self._estimate_message_media_tokens(
|
||||
messages,
|
||||
(model_value if isinstance(model_value, str) else "",),
|
||||
)
|
||||
).total
|
||||
total_chars: Final = (
|
||||
len(get_str_from_messages(messages))
|
||||
if isinstance(messages, list) and messages
|
||||
|
|
@ -1267,7 +1307,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else 0
|
||||
)
|
||||
|
||||
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
|
||||
estimated_text_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
|
||||
estimated_input_tokens: Final = estimated_text_tokens + estimated_media_tokens
|
||||
|
||||
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
|
||||
generates_no_output: Final = (
|
||||
|
|
@ -1283,7 +1324,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
max_tokens_estimate: Final = (
|
||||
0
|
||||
if generates_no_output
|
||||
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
|
||||
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
|
||||
else explicit_max_tokens
|
||||
if explicit_max_tokens is not None
|
||||
else configured_output_tokens
|
||||
|
|
@ -3752,27 +3793,77 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
descriptor_key=descriptor_key,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _estimate_message_media_tokens(cls, messages: object, models: tuple[str, ...] = ("",)) -> MediaTokenEstimate:
|
||||
if not isinstance(messages, list):
|
||||
return MediaTokenEstimate()
|
||||
estimates: Final = tuple(
|
||||
cls._estimate_content_media_tokens(message.get("content"), models)
|
||||
for message in _MESSAGE_CONTENT_OBJECTS.validate_python(messages)
|
||||
)
|
||||
return MediaTokenEstimate(
|
||||
image_tokens=sum(estimate.image_tokens for estimate in estimates),
|
||||
audio_tokens=sum(estimate.audio_tokens for estimate in estimates),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _estimate_content_media_tokens(cls, content: object, models: tuple[str, ...]) -> MediaTokenEstimate:
|
||||
if not isinstance(content, list):
|
||||
return MediaTokenEstimate()
|
||||
blocks: Final = _MESSAGE_CONTENT_OBJECTS.validate_python(content)
|
||||
return MediaTokenEstimate(
|
||||
image_tokens=sum(
|
||||
cls._estimate_image_block_tokens(block, models)
|
||||
for block in blocks
|
||||
if block.get("type") in ("image_url", "image")
|
||||
),
|
||||
audio_tokens=sum(
|
||||
cls._estimate_audio_block_tokens(block) for block in blocks if block.get("type") == "input_audio"
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_image_block_tokens(block: Mapping[str, object], models: tuple[str, ...]) -> int:
|
||||
image_url: Final = block.get("image_url")
|
||||
source: Final = block.get("source")
|
||||
image: Final = _callback_kwargs_mapping(image_url if image_url is not None else source)
|
||||
source_data: Final = image.get("data")
|
||||
url: Final = (
|
||||
image_url
|
||||
if isinstance(image_url, str)
|
||||
else image.get("url")
|
||||
if image_url is not None or image.get("type") == "url"
|
||||
else "data:image/unknown;base64," + source_data
|
||||
if image.get("type") == "base64" and isinstance(source_data, str)
|
||||
else ""
|
||||
)
|
||||
detail: Final = image.get("detail")
|
||||
return max(
|
||||
(
|
||||
estimate_image_tokens(
|
||||
model,
|
||||
url if isinstance(url, str) else "",
|
||||
detail if isinstance(detail, str) else "auto",
|
||||
)
|
||||
for model in models
|
||||
),
|
||||
default=DEFAULT_IMAGE_TOKEN_COUNT,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_audio_block_tokens(block: object) -> int:
|
||||
"""
|
||||
Token estimate for one ``input_audio`` content block.
|
||||
|
||||
When the block carries a base64 ``data`` payload, the estimate comes
|
||||
from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``),
|
||||
assuming the lowest reasonable audio bitrate so we never under-reserve
|
||||
for higher-quality recordings of the same duration.
|
||||
|
||||
When no payload is present (reference-only block or missing ``data``),
|
||||
falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``.
|
||||
"""
|
||||
if not isinstance(block, dict):
|
||||
audio: Final = _callback_kwargs_mapping(_callback_kwargs_mapping(block).get("input_audio"))
|
||||
b64_data: Final = audio.get("data")
|
||||
if not isinstance(b64_data, str) or not b64_data:
|
||||
return DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
input_audio: Final = block.get("input_audio")
|
||||
b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
|
||||
if b64_data and isinstance(b64_data, str):
|
||||
decoded_bytes: Final = len(b64_data) * 3 // 4
|
||||
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
|
||||
return DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
try:
|
||||
duration: Final = calculate_request_duration(base64.b64decode(b64_data, validate=True))
|
||||
if duration is not None and math.isfinite(duration) and duration > 0:
|
||||
return max(1, math.ceil(duration * _AUDIO_TOKENS_PER_SECOND))
|
||||
except (ValueError, binascii.Error):
|
||||
pass
|
||||
decoded_bytes: Final = len(b64_data) * 3 // 4
|
||||
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
|
||||
|
||||
@classmethod
|
||||
def _estimate_audio_content_tokens(cls, messages: object) -> int:
|
||||
|
|
@ -3799,7 +3890,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
Drop ``input_audio`` content blocks before passing ``messages`` to
|
||||
``token_counter``, which raises ``ValueError`` on them (no per-type
|
||||
handling, unlike images). The audio contribution is added back
|
||||
separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the
|
||||
separately via the duration estimate or unreadable-audio fallback so the rest of the
|
||||
message (text/images/tools) still gets counted accurately instead of
|
||||
the whole call falling back to the cheap char-count estimate.
|
||||
"""
|
||||
|
|
@ -3873,7 +3964,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
state: Final = data.get("state")
|
||||
return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}"
|
||||
|
||||
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
|
||||
def _estimate_precise_input_tokens(
|
||||
self,
|
||||
data: object,
|
||||
model: str | None,
|
||||
call_type: str | None = None,
|
||||
media_estimate: MediaTokenEstimate | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Model-aware input token estimate for the project ITPM reservation,
|
||||
using ``litellm.token_counter`` -- the same approach the
|
||||
|
|
@ -3937,7 +4034,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
messages, selected_text, countable_tools, countable_tool_choice = selected_inputs
|
||||
|
||||
audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
|
||||
audio_token_estimate: Final = (
|
||||
media_estimate.audio_tokens if media_estimate is not None else self._estimate_audio_content_tokens(messages)
|
||||
)
|
||||
countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
|
||||
|
||||
try:
|
||||
|
|
@ -3958,7 +4057,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
|
||||
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
|
||||
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
|
||||
data={**data, "messages": countable_messages},
|
||||
call_type=call_type,
|
||||
media_estimate=MediaTokenEstimate(image_tokens=media_estimate.image_tokens)
|
||||
if media_estimate is not None
|
||||
else None,
|
||||
)
|
||||
return estimated_input_tokens + audio_token_estimate
|
||||
|
||||
async def _reserve_project_io_tokens_or_raise(
|
||||
|
|
@ -3972,6 +4077,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type: str | None = None,
|
||||
endpoint_type: EndpointType = EndpointType.GENERIC,
|
||||
reuses_request_slot: bool = False,
|
||||
media_estimate: MediaTokenEstimate | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style
|
||||
|
|
@ -4003,9 +4109,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
data=data,
|
||||
min_configured_tpm_limit=min_configured_otpm_limit,
|
||||
call_type=call_type,
|
||||
media_estimate=media_estimate,
|
||||
)
|
||||
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=requested_model, call_type=call_type
|
||||
data=data,
|
||||
model=requested_model,
|
||||
call_type=call_type,
|
||||
media_estimate=media_estimate,
|
||||
)
|
||||
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
|
||||
estimated_output_tokens: Final = (
|
||||
|
|
@ -4323,6 +4433,35 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
has_tpm_limits: Final = bool(configured_tpm_limits)
|
||||
|
||||
needs_media_estimate: Final = (has_tpm_limits and self.tpm_reservation_enabled) or any(
|
||||
d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for d in descriptors
|
||||
)
|
||||
translated_request: Final = (
|
||||
self._translate_google_genai_native_request(data, call_type) if needs_media_estimate else None
|
||||
)
|
||||
media_messages: Final = (
|
||||
None
|
||||
if call_type
|
||||
in RESPONSES_API_CALL_TYPES
|
||||
+ EMBEDDING_API_CALL_TYPES
|
||||
+ TEXT_COMPLETION_API_CALL_TYPES
|
||||
+ EXTRACTION_API_CALL_TYPES
|
||||
else (translated_request if translated_request is not None else request_data).get("messages")
|
||||
)
|
||||
media_estimate: Final = (
|
||||
await offload_token_count(self._estimate_message_media_tokens)(
|
||||
_MESSAGE_CONTENT_OBJECTS.validate_python(media_messages),
|
||||
self._media_model_resolver(requested_model or "", user_api_key_dict.team_id),
|
||||
)
|
||||
if needs_media_estimate
|
||||
and isinstance(media_messages, list)
|
||||
and any(
|
||||
isinstance(message.get("content"), list)
|
||||
for message in _MESSAGE_CONTENT_OBJECTS.validate_python(media_messages)
|
||||
)
|
||||
else None
|
||||
)
|
||||
|
||||
# Populated on a successful combined-TPM reservation below, so the
|
||||
# project ITPM/OTPM block further down can roll it back if a
|
||||
# different bucket in the same request subsequently hits its
|
||||
|
|
@ -4366,6 +4505,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
media_estimate=media_estimate,
|
||||
),
|
||||
1,
|
||||
)
|
||||
|
|
@ -4390,6 +4530,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
raw_estimate=estimated_tokens,
|
||||
media_estimate=media_estimate,
|
||||
)
|
||||
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
|
||||
if not reuses_request_slot:
|
||||
|
|
@ -4453,6 +4594,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
tpm_reservation_scopes=tpm_reservation_scopes,
|
||||
tpm_reservation_amount=tpm_reservation_amount,
|
||||
media_estimate=media_estimate,
|
||||
call_type=call_type,
|
||||
endpoint_type=endpoint_type,
|
||||
reuses_request_slot=reuses_request_slot,
|
||||
|
|
|
|||
|
|
@ -13795,7 +13795,9 @@ class Router:
|
|||
]
|
||||
return named or self._get_wildcard_deployments(model_name=model_group)
|
||||
|
||||
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
|
||||
def resolved_litellm_models(
|
||||
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
|
||||
) -> tuple[str, ...]:
|
||||
"""The provider model strings `model_name` can actually be served by on this proxy.
|
||||
|
||||
`get_model_list` composes every channel the request path itself uses (exact name,
|
||||
|
|
@ -13808,6 +13810,11 @@ class Router:
|
|||
deployment behind it - so the fallback for an empty result is the caller's policy,
|
||||
never this function's.
|
||||
"""
|
||||
if use_base_model:
|
||||
return tuple(
|
||||
self._cost_map_backend_model(Deployment.model_validate(deployment))
|
||||
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
)
|
||||
return tuple(
|
||||
litellm_model
|
||||
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import (
|
|||
_get_extrapolating_count_function,
|
||||
_get_tiktoken_count_function,
|
||||
calculate_img_tokens,
|
||||
estimate_image_tokens,
|
||||
high_detail_image_token_upper_bound,
|
||||
offload_token_count,
|
||||
)
|
||||
|
|
@ -2025,6 +2026,111 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_
|
|||
assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound()
|
||||
|
||||
|
||||
# Image policies verified 2026-10-24: https://developers.openai.com/api/docs/guides/images-vision
|
||||
@pytest.mark.parametrize(
|
||||
("model", "width", "height", "detail", "expected"),
|
||||
[
|
||||
("gpt-4o", 16, 16, "high", 255),
|
||||
("gpt-4.1", 1024, 1024, "high", 765),
|
||||
("gpt-4o-mini", 1024, 1024, "high", 25501),
|
||||
("gpt-5.1", 1024, 1024, "auto", 630),
|
||||
("gpt-5", 1024, 1024, "low", 70),
|
||||
("o1", 1024, 1024, "high", 675),
|
||||
("o1-pro", 1024, 1024, "high", 675),
|
||||
("o3", 1024, 1024, "low", 75),
|
||||
("openai/gpt-4o-2024-08-06", 2048, 4096, "auto", 1105),
|
||||
("azure/gpt-4.1", 1024, 1024, "low", 85),
|
||||
("openrouter/openai/gpt-4o-mini", 16, 16, "low", 2833),
|
||||
("gpt-6-astra", 1024, 1024, "high", 1229),
|
||||
("gpt-6-astra", 2048, 2048, "high", 3000),
|
||||
("gpt-6-astra", 4096, 512, "high", 2458),
|
||||
("gpt-6-astra", 2048, 2048, "auto", 4916),
|
||||
("gpt-6-astra", 2048, 2048, "low", 308),
|
||||
("gpt-5.6-sol", 4096, 512, "high", 615),
|
||||
("gpt-5.6-terra", 2048, 2048, "original", 4916),
|
||||
("gpt-5.6-luna", 2048, 2048, "auto", 4916),
|
||||
("gpt-5.5", 6400, 6400, "original", 12000),
|
||||
("gpt-5.4", 2048, 2048, "low", 4916),
|
||||
("gpt-5.4-mini", 2048, 2048, "auto", 3000),
|
||||
("gpt-5.4-nano", 6400, 6400, "original", 12000),
|
||||
("gpt-5.2", 4096, 512, "auto", 615),
|
||||
("gpt-4.1-mini-2025-04-14", 1024, 1024, "low", 1659),
|
||||
("gpt-6-astra", 16, 16, "original", 2),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_uses_model_resolution_and_detail(
|
||||
model: str, width: int, height: int, detail: str, expected: int
|
||||
) -> None:
|
||||
assert estimate_image_tokens(model, _png_data_url(width, height), detail) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "data", "detail"),
|
||||
[
|
||||
("unregistered-model", _png_data_url(1024, 1024), "high"),
|
||||
("gpt-4.1-unrecognized", _png_data_url(1024, 1024), "high"),
|
||||
("other/gpt-4o", _png_data_url(1024, 1024), "high"),
|
||||
("gpt-4.1-nano", _png_data_url(1024, 1024), "high"),
|
||||
("gpt-4o", _png_data_url(1024, 1024), "original"),
|
||||
("gpt-5.2", _png_data_url(1024, 1024), "original"),
|
||||
("gpt-6-astra", _png_data_url(1024, 1024), "invalid"),
|
||||
("gpt-4o", "", "high"),
|
||||
("gpt-4o", "data:image/png;base64,not base64!", "high"),
|
||||
("gpt-4o", "data:image/png;base64," + base64.b64encode(b"unknown format").decode(), "high"),
|
||||
("gpt-4o", "data:image/jpeg;base64,/9j/", "high"),
|
||||
("gpt-4o", "data:image/webp;base64," + base64.b64encode(b"RIFF\0\0\0\0WEBPbad!").decode(), "high"),
|
||||
("gpt-4o", _png_data_url(0, 1024), "high"),
|
||||
("gpt-4o", "data:image/png;base64," + base64.b64encode(b"\x89PNG\r\n\x1a\n").decode(), "high"),
|
||||
],
|
||||
)
|
||||
def test_image_estimate_falls_back_when_policy_or_dimensions_are_unavailable(
|
||||
model: str, data: str, detail: str
|
||||
) -> None:
|
||||
assert estimate_image_tokens(model, data, detail) == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "detail", "expected"), [
|
||||
("gpt-4o", "high", litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT),
|
||||
("gpt-6-astra", "original", litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT),
|
||||
("gpt-4o-mini", "low", 2833),
|
||||
])
|
||||
def test_image_estimate_does_not_fetch_remote_media(model: str, detail: str, expected: int) -> None:
|
||||
with patch("httpx.Client.send") as send:
|
||||
assert estimate_image_tokens(model, "https://image.invalid/large.png", detail) == expected
|
||||
send.assert_not_called()
|
||||
|
||||
|
||||
def test_image_estimate_bounds_decoding_independent_of_payload_size() -> None:
|
||||
header: Final = base64.b64decode(_png_data_url(1024, 1024).split(",", 1)[1])
|
||||
data: Final = "data:image/png;base64," + base64.b64encode(header + b"\0" * 4_000_000).decode()
|
||||
with patch("base64.b64decode", wraps=base64.b64decode) as decode_header:
|
||||
assert estimate_image_tokens("gpt-6-astra", data, "high") == 1229
|
||||
assert max(len(call.args[0]) for call in decode_header.call_args_list) <= 65_536
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", [
|
||||
b"GIF89a" + (1024).to_bytes(2, "little") + (512).to_bytes(2, "little"),
|
||||
b"\xff\xd8\xff\xc0\x00\x0b\x08" + (512).to_bytes(2, "big") + (1024).to_bytes(2, "big"),
|
||||
b"RIFF\0\0\0\0WEBPVP8X" + b"\0" * 8 + (1023).to_bytes(3, "little") + (511).to_bytes(3, "little"),
|
||||
b"RIFF\0\0\0\0WEBPVP8 " + b"\0" * 10 + (1024).to_bytes(2, "little") + (512).to_bytes(2, "little"),
|
||||
b"RIFF\0\0\0\0WEBPVP8L" + b"\0" * 5 + (1023 | (511 << 14)).to_bytes(4, "little"),
|
||||
])
|
||||
def test_image_estimate_reads_supported_image_headers(header: bytes) -> None:
|
||||
data: Final = "data:image/octet-stream;base64," + base64.b64encode(header).decode()
|
||||
assert estimate_image_tokens("gpt-4o", data, "high") == estimate_image_tokens(
|
||||
"gpt-4o", _png_data_url(1024, 512), "high"
|
||||
)
|
||||
|
||||
|
||||
def test_image_estimate_falls_back_when_dimensions_are_beyond_header_budget() -> None:
|
||||
jpeg: Final = (
|
||||
b"\xff\xd8\xff\xe1\xff\xff" + b"\0" * 65533
|
||||
+ b"\xff\xc0\x00\x0b\x08" + (512).to_bytes(2, "big") + (1024).to_bytes(2, "big")
|
||||
)
|
||||
data: Final = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
|
||||
assert estimate_image_tokens("gpt-4o", data, "high") == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT
|
||||
|
||||
|
||||
HUB_SETUP_SCRIPT: Final = """
|
||||
import json
|
||||
import sys
|
||||
|
|
|
|||
|
|
@ -15,19 +15,27 @@ Redis.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import wave
|
||||
from functools import partial
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm import Router
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
PROJECT_ITPM_DESCRIPTOR_KEY,
|
||||
PROJECT_OTPM_DESCRIPTOR_KEY,
|
||||
RateLimitedModel,
|
||||
_AUDIO_BYTES_PER_TOKEN,
|
||||
_resolve_media_models,
|
||||
PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler,
|
||||
)
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
|
|
@ -53,6 +61,197 @@ def rate_limiter():
|
|||
return handler, cache
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("media_type", ["image_url", "image", "input_audio"])
|
||||
async def test_combined_tpm_rejects_media_before_dispatch(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], media_type: str,
|
||||
) -> None:
|
||||
handler, cache = rate_limiter
|
||||
media: Final = {
|
||||
"type": media_type,
|
||||
"image_url": {"url": "https://example.invalid/image.png", "detail": "high"},
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": "AAAA"},
|
||||
"input_audio": {"data": "AAAA", "format": "wav"},
|
||||
}
|
||||
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-media-tpm"), tpm_limit=100)
|
||||
data: Final = {
|
||||
"model": "unknown-deployment",
|
||||
"messages": [{"role": "user", "content": [media]}],
|
||||
"max_tokens": 16,
|
||||
}
|
||||
with pytest.raises(ProxyRateLimitError) as exc:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion",
|
||||
)
|
||||
assert exc.value.status_code == 429
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_cap", [None, 0, 16])
|
||||
@pytest.mark.parametrize("configured_output", [None, 32])
|
||||
def test_combined_tpm_media_is_bounded_and_not_contentless(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache],
|
||||
output_cap: int | None, configured_output: int | None,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
for image_data in ("AAAA", "A" * 4_000_000):
|
||||
data: Final = {
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {
|
||||
"url": "data:image/png;base64," + image_data,
|
||||
}},
|
||||
]}],
|
||||
"n": 2,
|
||||
**({"max_tokens": output_cap} if output_cap is not None else {}),
|
||||
}
|
||||
input_tokens, output_tokens = handler._estimate_input_and_output_tokens(
|
||||
data, call_type="acompletion", configured_output_tokens=configured_output,
|
||||
)
|
||||
expected_output: Final = (
|
||||
output_cap if output_cap is not None else configured_output
|
||||
if configured_output is not None else max(
|
||||
DEFAULT_IMAGE_TOKEN_COUNT, handler.no_max_tokens_output_floor(None),
|
||||
)
|
||||
)
|
||||
assert input_tokens == DEFAULT_IMAGE_TOKEN_COUNT
|
||||
assert output_tokens == expected_output * 2
|
||||
|
||||
|
||||
def test_combined_tpm_preserves_text_and_counts_each_media_block_once(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache],
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
audio: Final = {
|
||||
"type": "input_audio", "input_audio": {"data": "A" * 4_000_000, "format": "wav"},
|
||||
}
|
||||
image: Final = {"type": "image_url", "image_url": "https://example.invalid/image.png"}
|
||||
data: Final = {
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "text", "text": "abcd" * 10}, image, image, audio,
|
||||
]}], "max_tokens": 0,
|
||||
}
|
||||
assert handler._estimate_input_and_output_tokens(data) == (
|
||||
10 + 2 * DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio), 0,
|
||||
)
|
||||
|
||||
|
||||
def test_project_itpm_fallback_counts_chat_audio_once(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache],
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
audio: Final = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}
|
||||
data: Final = {
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "text", "text": "abcd" * 10}, audio,
|
||||
{"type": "image_url", "image_url": {"url": "https://example.invalid/image.png"}},
|
||||
{"type": "unsupported_counter_block"},
|
||||
]}],
|
||||
}
|
||||
assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == (
|
||||
10 + DEFAULT_IMAGE_TOKEN_COUNT + handler._estimate_audio_block_tokens(audio)
|
||||
)
|
||||
|
||||
|
||||
def _audio_message(seconds: int, sample_rate: int, channels: int) -> dict[str, object]:
|
||||
output: Final = io.BytesIO()
|
||||
with wave.open(output, "wb") as wav:
|
||||
wav.setnchannels(channels)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(b"\x00\x00" * seconds * sample_rate * channels)
|
||||
return {"role": "user", "content": [{
|
||||
"type": "input_audio", "input_audio": {
|
||||
"data": base64.b64encode(output.getvalue()).decode(), "format": "wav",
|
||||
},
|
||||
}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate,channels", [(8000, 1), (24000, 1), (48000, 2)])
|
||||
def test_audio_estimates_follow_duration_instead_of_encoding_size(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], sample_rate: int, channels: int,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
# OpenAI user-audio estimate, checked 2026-10-24:
|
||||
# https://developers.openai.com/api/docs/guides/voice-latency-cost?voice-api=realtime
|
||||
for seconds in (1, 31):
|
||||
message: Final = _audio_message(seconds, sample_rate, channels)
|
||||
data: Final = {"model": "gpt-audio-1.5", "messages": [message], "max_tokens": 0}
|
||||
assert handler._estimate_input_and_output_tokens(data) == (seconds * 10, 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("seconds,admitted", [(1, True), (31, False)])
|
||||
async def test_audio_duration_drives_combined_and_project_admission(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], seconds: int, admitted: bool,
|
||||
) -> None:
|
||||
handler, cache = rate_limiter
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-duration"), tpm_limit=100, project_id="audio-project",
|
||||
project_metadata={"model_itpm_limit": {"gpt-audio-1.5": 100}},
|
||||
)
|
||||
data: Final = {"model": "gpt-audio-1.5", "messages": [_audio_message(seconds, 24000, 1)], "max_tokens": 16}
|
||||
if not admitted:
|
||||
with pytest.raises(ProxyRateLimitError):
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
return
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
assert stash.reserved_tokens == 26
|
||||
assert 10 <= stash.itpm_reserved_tokens < 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("detail,admitted", [("low", True), ("high", False), ("auto", False)])
|
||||
@pytest.mark.parametrize("route", ["alias", "azure", "mixed"])
|
||||
async def test_model_specific_images_resolve_routes_before_admission(
|
||||
detail: str, admitted: bool, route: str,
|
||||
) -> None:
|
||||
from PIL import Image
|
||||
|
||||
output: Final = io.BytesIO()
|
||||
Image.new("RGB", (1024, 1024)).save(output, format="PNG")
|
||||
image: Final = "data:image/png;base64," + base64.b64encode(output.getvalue()).decode()
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "vision", "litellm_params": {"model": "openai/gpt-5.1", "api_key": "test"}},
|
||||
{"model_name": "azure", "litellm_params": {
|
||||
"model": "azure/custom-deployment", "base_model": "gpt-4o-mini",
|
||||
"api_key": "test", "api_base": "https://example.invalid", "api_version": "2024-12-01-preview",
|
||||
}, "model_info": {"base_model": "gpt-5.1"}},
|
||||
{"model_name": "mixed", "litellm_params": {"model": "openai/gpt-5.1", "api_key": "test"}},
|
||||
{"model_name": "mixed", "litellm_params": {"model": "openai/gpt-4o", "api_key": "test"}},
|
||||
], model_group_alias={"alias": "vision"},
|
||||
)
|
||||
cache: Final = DualCache()
|
||||
handler: Final = RateLimitHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), media_model_resolver=partial(_resolve_media_models, router),
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-image-policy"), tpm_limit=500)
|
||||
data: Final = {"model": route, "messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": image, "detail": detail}},
|
||||
]}], "max_tokens": 16}
|
||||
if not admitted:
|
||||
with pytest.raises(ProxyRateLimitError):
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
return
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
assert stash.reserved_tokens == (85 if route == "mixed" else 70) + 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate,channels", [(8000, 1), (48000, 2)])
|
||||
def test_project_fallback_reuses_audio_duration_once(
|
||||
rate_limiter: tuple[RateLimitHandler, DualCache], sample_rate: int, channels: int,
|
||||
) -> None:
|
||||
handler, _ = rate_limiter
|
||||
data: Final = {"model": "unknown-deployment", "messages": [
|
||||
_audio_message(31, sample_rate, channels),
|
||||
{"role": "user", "content": [{"type": "unsupported_counter_block"}]},
|
||||
]}
|
||||
assert handler._estimate_precise_input_tokens(data, "unknown-deployment", "acompletion") == 310
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_request_stash():
|
||||
token = _request_stash.set(None)
|
||||
|
|
|
|||
|
|
@ -30149,3 +30149,21 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items():
|
|||
assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True
|
||||
assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False
|
||||
assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("info_base,params_base", [(None, None), (None, "gpt-5.1"), ("gpt-6-astra", "gpt-5.1")])
|
||||
def test_resolved_litellm_models_can_use_configured_base_model(
|
||||
info_base: str | None, params_base: str | None,
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[{
|
||||
"model_name": "deployment", "litellm_params": {
|
||||
"model": "azure/custom-name", "api_key": "test", "api_base": "https://example.invalid",
|
||||
"api_version": "2024-12-01-preview", **({"base_model": params_base} if params_base else {}),
|
||||
}, "model_info": {"base_model": info_base},
|
||||
}], model_group_alias={"alias": "deployment"},
|
||||
)
|
||||
assert router.resolved_litellm_models("alias") == ("azure/custom-name",)
|
||||
assert router.resolved_litellm_models("alias", use_base_model=True) == (
|
||||
info_base or params_base or "azure/custom-name",
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue