From 4706acef958a726f1407ad5413ea2db896d6c825 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 03:48:06 -0700 Subject: [PATCH] fix(passthrough): charge remote high-detail images at the high-detail upper bound --- litellm/litellm_core_utils/token_counter.py | 7 ++++++ .../openai_passthrough_logging_handler.py | 4 ++-- ruff-strict-budget.json | 4 ++-- .../litellm_core_utils/test_token_counter.py | 22 ++++++++++++++++++- .../test_azure_passthrough_transformation.py | 4 ++-- ...test_openai_passthrough_logging_handler.py | 8 +++---- type-discipline-budget.json | 6 ++--- 7 files changed, 41 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 3732ffd734c..4c66d878f14 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -172,6 +172,13 @@ def calculate_tiles_needed( return total_tiles +def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int: + largest_tile_count: Final = calculate_tiles_needed( + MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES + ) + return base_tokens + (base_tokens * 2) * largest_tile_count + + def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]: return struct.unpack(fmt, buffer) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 6a142d08be5..206002d13f7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -13,11 +13,11 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -133,7 +133,7 @@ def count_relayed_prompt_tokens(model: str, messages: Sequence[Mapping[str, obje ] return ( litellm.token_counter(model=model, messages=local_messages) - + DEFAULT_IMAGE_TOKEN_COUNT * remote_high_detail_images + + high_detail_image_token_upper_bound() * remote_high_detail_images ) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 574bf01cc34..134eec578bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2904 + "limit": 2900 }, "C401": { "limit": 8 @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 110 + "limit": 108 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 4694fa8fbed..1898fd57220 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1,5 +1,6 @@ #### What this tests #### # This tests litellm.token_counter.token_counter() function +import base64 import importlib import time import traceback @@ -14,7 +15,11 @@ import litellm from litellm import create_pretrained_tokenizer, decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.litellm_core_utils.token_counter import _get_tiktoken_count_function +from litellm.litellm_core_utils.token_counter import ( + _get_tiktoken_count_function, + calculate_img_tokens, + high_detail_image_token_upper_bound, +) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from tests.large_text import text from tests.test_litellm.litellm_core_utils.messages_with_counts import ( @@ -1412,3 +1417,18 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): assert _count_user_content([prompt, named]) == _count_user_content( [prompt, {"type": "text", "text": "report.pdf"}] ) + + +def _png_data_url(width: int, height: int) -> str: + ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") + return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() + + +@pytest.mark.parametrize(("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)]) +def test_high_detail_image_token_upper_bound_covers_every_image_size(width: int, height: int) -> None: + assert calculate_img_tokens(_png_data_url(width, height), mode="high") <= high_detail_image_token_upper_bound() + + +def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_image() -> None: + assert calculate_img_tokens(_png_data_url(2000, 768), mode="high") == high_detail_image_token_upper_bound() + assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 5d9e4312b5e..8ffd632fe85 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -5,9 +5,9 @@ import httpx import pytest import litellm -from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig from litellm.types.utils import ModelResponse @@ -179,7 +179,7 @@ def test_azure_passthrough_streaming_chunks_count_remote_image_prompt_tokens_wit text_only_messages = [{"role": "user", "content": [{"type": "text", "text": "Describe this"}]}] assert isinstance(response, ModelResponse) assert response.usage.prompt_tokens == ( - litellm.token_counter(model="gpt-4.1-mini", messages=text_only_messages) + DEFAULT_IMAGE_TOKEN_COUNT + litellm.token_counter(model="gpt-4.1-mini", messages=text_only_messages) + high_detail_image_token_upper_bound() ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 23c26a0b8a3..0ddafd20e68 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -8,8 +8,8 @@ import pytest import litellm -from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, count_relayed_prompt_tokens, @@ -2074,12 +2074,12 @@ def test_count_relayed_prompt_tokens_keeps_a_low_detail_remote_image_at_the_base assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == litellm.token_counter( model="gpt-4.1-mini", messages=messages ) - assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) < DEFAULT_IMAGE_TOKEN_COUNT + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) < high_detail_image_token_upper_bound() -def test_count_relayed_prompt_tokens_estimates_only_the_remote_high_detail_image(): +def test_count_relayed_prompt_tokens_charges_only_the_remote_high_detail_image_at_the_upper_bound(): messages = _image_messages(UNREACHABLE_IMAGE_URL, "high") assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == ( - litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + DEFAULT_IMAGE_TOKEN_COUNT + litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + high_detail_image_token_upper_bound() ) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 3a4abfbc20a..b4c17ffd47b 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22156 + "limit": 22146 }, "LIT002": { "limit": 26745 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16434 + "limit": 16422 }, "LIT011": { - "limit": 5504 + "limit": 5502 }, "LIT012": { "limit": 4486