diff --git a/litellm/constants.py b/litellm/constants.py index d1daf2ee2d1..78670aec38a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2,6 +2,7 @@ import os import re import sys from enum import Enum +from fractions import Fraction from types import MappingProxyType from typing import Final, Literal @@ -533,6 +534,10 @@ DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESP DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20) MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768) MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000) +# OpenAI's "original" image detail, https://developers.openai.com/api/docs/guides/images-vision (2026-10-19) +OPENAI_IMAGE_PATCH_SIZE_PX: Final = 32 +OPENAI_IMAGE_MAX_PATCHES: Final = 30_000 +OPENAI_IMAGE_PATCH_TOKEN_MULTIPLIER: Final = Fraction(6, 5) # tiktoken's BPE merge loop is quadratic in the length of a single regex piece, so a long run of one # repeated character (dot leaders, whitespace, zero-padded base64) can take minutes on a multi-MB payload. # Encoding in chunks makes the cost linear, at a drift of at most ~1 token per chunk boundary. The upper diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 90e79d4d853..28170ea1667 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -6,7 +6,7 @@ import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast import anyio import anyio.lowlevel @@ -29,6 +29,9 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_IMAGE_MAX_PATCHES, + OPENAI_IMAGE_PATCH_SIZE_PX, + OPENAI_IMAGE_PATCH_TOKEN_MULTIPLIER, PDF_DATA_URL_PREFIX, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, @@ -67,6 +70,9 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import Message, SelectTokenizerResponse +ImageDetail: TypeAlias = Literal["low", "high", "auto", "original"] +IMAGE_DETAILS: Final[frozenset[ImageDetail]] = frozenset(("low", "high", "auto", "original")) + def get_modified_max_tokens( model: str, @@ -195,6 +201,11 @@ def calculate_tiles_needed( return total_tiles +def original_detail_image_tokens(width: int, height: int) -> int: + patches: Final = math.ceil(width / OPENAI_IMAGE_PATCH_SIZE_PX) * math.ceil(height / OPENAI_IMAGE_PATCH_SIZE_PX) + return math.ceil(min(patches, OPENAI_IMAGE_MAX_PATCHES) * OPENAI_IMAGE_PATCH_TOKEN_MULTIPLIER) + + def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int: largest_tile_count: Final = calculate_tiles_needed( MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES @@ -309,7 +320,7 @@ def get_image_dimensions( def calculate_img_tokens( data, - mode: Literal["low", "high", "auto"] = "auto", + mode: ImageDetail = "auto", base_tokens: int = 85, # openai default - https://openai.com/pricing use_default_image_token_count: bool = False, ): @@ -318,7 +329,7 @@ def calculate_img_tokens( Args: data (str): The URL or base64 encoded string of the image. - mode (Literal["low", "high", "auto"]): The mode to use for calculating the number of tokens. + mode (ImageDetail): The mode to use for calculating the number of tokens. base_tokens (int): The base number of tokens for an image. use_default_image_token_count (bool): When True, will NOT make a GET request to the image URL and instead return the default image dimensions. @@ -330,6 +341,8 @@ def calculate_img_tokens( return DEFAULT_IMAGE_TOKEN_COUNT if mode == "low" or mode == "auto": return base_tokens + elif mode == "original": + return original_detail_image_tokens(*get_image_dimensions(data=data)) elif mode == "high": # Run the async function using the helper width, height = get_image_dimensions( @@ -790,8 +803,8 @@ def _count_image_tokens( """ if isinstance(image_url, dict): detail: Final = image_url.get("detail", "auto") - if detail not in ["low", "high", "auto"]: - raise ValueError(f"Invalid detail value: {detail}. Expected 'low', 'high', or 'auto'.") + if detail not in IMAGE_DETAILS: + raise ValueError(f"Invalid detail value: {detail}. Expected 'low', 'high', 'auto', or 'original'.") url: Final = image_url.get("url") if not url: raise ValueError("Missing required key 'url' in image_url dict.") diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 88af656ad2e..01ebf84226b 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2025,6 +2025,25 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_ assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() +def _original_detail_message(width: int, height: int) -> list[dict[str, object]]: + image: Final = {"url": _png_data_url(width, height), "detail": "original"} + return [{"role": "user", "content": [{"type": "image_url", "image_url": image}]}] + + +def test_original_detail_image_is_counted_at_its_full_resolution() -> None: + large_image_tokens: Final = token_counter(messages=_original_detail_message(4000, 3000)) + + assert large_image_tokens > high_detail_image_token_upper_bound() + assert token_counter(messages=_original_detail_message(512, 512)) < large_image_tokens + + +def test_original_detail_image_tokens_stop_growing_at_the_largest_accepted_image() -> None: + oversized: Final = calculate_img_tokens(_png_data_url(20_000, 20_000), mode="original") + + assert calculate_img_tokens(_png_data_url(40_000, 40_000), mode="original") == oversized + assert calculate_img_tokens(_png_data_url(4000, 3000), mode="original") < oversized + + HUB_SETUP_SCRIPT: Final = """ import json import sys diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 568a05e8ff4..078340d9d5a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1,4 +1,5 @@ import ast, asyncio, importlib, time +import base64 import contextlib import copy import functools @@ -38,6 +39,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, @@ -8101,6 +8103,33 @@ async def test_aresponses_enforces_context_window_pre_call_check(): ) +@pytest.mark.asyncio +async def test_aresponses_counts_an_original_detail_image_against_the_context_window(): + ihdr: Final = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + (4000).to_bytes(4, "big") * 2 + large_image: Final = "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vision", + "litellm_params": {"model": "gpt-5.5", "mock_response": "hi"}, + "model_info": {"max_input_tokens": high_detail_image_token_upper_bound() * 2}, + } + ], + enable_pre_call_checks=True, + ) + + with pytest.raises(litellm.ContextWindowExceededError): + await router.aresponses( + model="vision", + input=[ + { + "role": "user", + "content": [{"type": "input_image", "image_url": large_image, "detail": "original"}], + } + ], + ) + + def test_get_deployment_model_info_base_model_flow(): """Test that get_deployment_model_info correctly handles the base model flow""" from unittest.mock import patch