From 63a1681cbbcf1dcc0ca0eca4bf9e911f4b55c2f2 Mon Sep 17 00:00:00 2001 From: Jorik Date: Mon, 19 Oct 2026 15:48:28 +0200 Subject: [PATCH] fix(token_counter): count images sent with detail original OpenAI and Azure OpenAI accept "original" as an image detail value, but token_counter only knew low, high and auto and raised on it. The router then logged an ERROR per request and skipped the context window check. Count original images with OpenAI's patch formula at full resolution, capped at the largest image any model accepts. --- litellm/constants.py | 5 ++++ litellm/litellm_core_utils/token_counter.py | 23 +++++++++++---- .../litellm_core_utils/test_token_counter.py | 19 ++++++++++++ tests/unit/test_router/test_router.py | 29 +++++++++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index da95b97036a..fcb4d2c2511 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2,6 +2,7 @@ import os import re import sys from enum import Enum +from fractions import Fraction from types import MappingProxyType from typing import Final, Literal @@ -504,6 +505,10 @@ DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESP DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20) MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768) MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000) +# OpenAI's "original" image detail, https://developers.openai.com/api/docs/guides/images-vision (2026-10-19) +OPENAI_IMAGE_PATCH_SIZE_PX: Final = 32 +OPENAI_IMAGE_MAX_PATCHES: Final = 30_000 +OPENAI_IMAGE_PATCH_TOKEN_MULTIPLIER: Final = Fraction(6, 5) # tiktoken's BPE merge loop is quadratic in the length of a single regex piece, so a long run of one # repeated character (dot leaders, whitespace, zero-padded base64) can take minutes on a multi-MB payload. # Encoding in chunks makes the cost linear, at a drift of at most ~1 token per chunk boundary. The upper diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 7ac3e9c80c5..707b1badabb 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -6,7 +6,7 @@ import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast import anyio import anyio.lowlevel @@ -29,6 +29,9 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_IMAGE_MAX_PATCHES, + OPENAI_IMAGE_PATCH_SIZE_PX, + OPENAI_IMAGE_PATCH_TOKEN_MULTIPLIER, PDF_DATA_URL_PREFIX, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, @@ -63,6 +66,9 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import Message, SelectTokenizerResponse +ImageDetail: TypeAlias = Literal["low", "high", "auto", "original"] +IMAGE_DETAILS: Final[frozenset[ImageDetail]] = frozenset(("low", "high", "auto", "original")) + def get_modified_max_tokens( model: str, @@ -191,6 +197,11 @@ def calculate_tiles_needed( return total_tiles +def original_detail_image_tokens(width: int, height: int) -> int: + patches: Final = math.ceil(width / OPENAI_IMAGE_PATCH_SIZE_PX) * math.ceil(height / OPENAI_IMAGE_PATCH_SIZE_PX) + return math.ceil(min(patches, OPENAI_IMAGE_MAX_PATCHES) * OPENAI_IMAGE_PATCH_TOKEN_MULTIPLIER) + + def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int: largest_tile_count: Final = calculate_tiles_needed( MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES @@ -305,7 +316,7 @@ def get_image_dimensions( def calculate_img_tokens( data, - mode: Literal["low", "high", "auto"] = "auto", + mode: ImageDetail = "auto", base_tokens: int = 85, # openai default - https://openai.com/pricing use_default_image_token_count: bool = False, ): @@ -314,7 +325,7 @@ def calculate_img_tokens( Args: data (str): The URL or base64 encoded string of the image. - mode (Literal["low", "high", "auto"]): The mode to use for calculating the number of tokens. + mode (ImageDetail): The mode to use for calculating the number of tokens. base_tokens (int): The base number of tokens for an image. use_default_image_token_count (bool): When True, will NOT make a GET request to the image URL and instead return the default image dimensions. @@ -326,6 +337,8 @@ def calculate_img_tokens( return DEFAULT_IMAGE_TOKEN_COUNT if mode == "low" or mode == "auto": return base_tokens + elif mode == "original": + return original_detail_image_tokens(*get_image_dimensions(data=data)) elif mode == "high": # Run the async function using the helper width, height = get_image_dimensions( @@ -748,8 +761,8 @@ def _count_image_tokens( """ if isinstance(image_url, dict): detail: Final = image_url.get("detail", "auto") - if detail not in ["low", "high", "auto"]: - raise ValueError(f"Invalid detail value: {detail}. Expected 'low', 'high', or 'auto'.") + if detail not in IMAGE_DETAILS: + raise ValueError(f"Invalid detail value: {detail}. Expected 'low', 'high', 'auto', or 'original'.") url: Final = image_url.get("url") if not url: raise ValueError("Missing required key 'url' in image_url dict.") diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 84c139e867b..47c0b703f2e 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1635,6 +1635,25 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_ assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() +def _original_detail_message(width: int, height: int) -> list[dict[str, object]]: + image: Final = {"url": _png_data_url(width, height), "detail": "original"} + return [{"role": "user", "content": [{"type": "image_url", "image_url": image}]}] + + +def test_original_detail_image_is_counted_at_its_full_resolution() -> None: + large_image_tokens: Final = token_counter(messages=_original_detail_message(4000, 3000)) + + assert large_image_tokens > high_detail_image_token_upper_bound() + assert token_counter(messages=_original_detail_message(512, 512)) < large_image_tokens + + +def test_original_detail_image_tokens_stop_growing_at_the_largest_accepted_image() -> None: + oversized: Final = calculate_img_tokens(_png_data_url(20_000, 20_000), mode="original") + + assert calculate_img_tokens(_png_data_url(40_000, 40_000), mode="original") == oversized + assert calculate_img_tokens(_png_data_url(4000, 3000), mode="original") < oversized + + HUB_SETUP_SCRIPT: Final = """ import json import sys diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 1dec4de45f4..4382f95ea4b 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1,4 +1,5 @@ import ast, asyncio, importlib, time +import base64 import contextlib import copy import functools @@ -38,6 +39,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, @@ -7813,6 +7815,33 @@ async def test_aresponses_enforces_context_window_pre_call_check(): ) +@pytest.mark.asyncio +async def test_aresponses_counts_an_original_detail_image_against_the_context_window(): + ihdr: Final = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + (4000).to_bytes(4, "big") * 2 + large_image: Final = "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vision", + "litellm_params": {"model": "gpt-5.5", "mock_response": "hi"}, + "model_info": {"max_input_tokens": high_detail_image_token_upper_bound() * 2}, + } + ], + enable_pre_call_checks=True, + ) + + with pytest.raises(litellm.ContextWindowExceededError): + await router.aresponses( + model="vision", + input=[ + { + "role": "user", + "content": [{"type": "input_image", "image_url": large_image, "detail": "original"}], + } + ], + ) + + def test_get_deployment_model_info_base_model_flow(): """Test that get_deployment_model_info correctly handles the base model flow""" from unittest.mock import patch