From 228914b0bd26ba796f84321e88a54ed22965e287 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 20:51:43 +0000 Subject: [PATCH] fix: count anthropic image content blocks in token_counter instead of raising Anthropic-format image blocks ({"type": "image", "source": ...}) hit the else branch in _count_content_list and raised, breaking router pre-call checks and cost tracking for anthropic messages traffic carrying images. Count them with a dimension-based estimate (reusing calculate_img_tokens via the existing image_url path) instead of a flat default, since billed tokens scale with image dimensions. Falls back to DEFAULT_IMAGE_TOKEN_COUNT when the source can't be parsed. --- litellm/litellm_core_utils/token_counter.py | 53 ++++++++++++++- .../litellm_core_utils/test_token_counter.py | 67 +++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 17f3dea72ec..3272f761a06 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -682,6 +682,51 @@ def _count_anthropic_content( return tokens +def _anthropic_image_source_to_data_uri(source: object) -> str | None: + """Convert an Anthropic image source to the URL / data-URI form ``calculate_img_tokens`` expects.""" + if not isinstance(source, dict): + return None + source_type: Final = source.get("type") + if source_type == "url": + url: Final = source.get("url") + return url if isinstance(url, str) and url else None + if source_type == "base64": + raw_data: Final = source.get("data") + if not isinstance(raw_data, str) or not raw_data: + return None + media_type: Final = source.get("media_type") or "image/png" + return f"data:{media_type};base64,{raw_data}" + return None + + +def _count_anthropic_image_tokens( + source: object, + use_default_image_token_count: bool, +) -> int: + """ + Count tokens for an Anthropic image content block source + ({"type": "base64", "media_type": ..., "data": ...} or {"type": "url", "url": ...}). + + Converts the source to the URL / data-URI form understood by + _count_image_tokens so the dimension-aware estimate applies; falls back to + DEFAULT_IMAGE_TOKEN_COUNT when the source cannot be interpreted. + """ + data: Final = _anthropic_image_source_to_data_uri(source) + if data is None: + return DEFAULT_IMAGE_TOKEN_COUNT + try: + # mode="high" applies the dimension-based tile math; "auto" resolves to + # the flat low-detail estimate, and anthropic image billing always + # scales with dimensions (there is no detail tier to select) + return calculate_img_tokens( + data=data, + mode="high", + use_default_image_token_count=use_default_image_token_count, + ) + except (ValueError, TypeError, struct.error): + return DEFAULT_IMAGE_TOKEN_COUNT + + def _count_content_list( count_function: TokenCounterFunction, content_list: OpenAIMessageContent, @@ -701,6 +746,12 @@ def _count_content_list( elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) + elif c["type"] == "image": + # Anthropic-format image block ({"type": "image", "source": ...}). + # Reuse the dimension-aware image_url counting by converting the + # source to a URL / data URI; a flat default would under-count + # large images since billed tokens scale with dimensions. + num_tokens += _count_anthropic_image_tokens(c.get("source"), use_default_image_token_count) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -729,7 +780,7 @@ def _count_content_list( content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__ raise ValueError( f"Invalid content item type: {content_type}. " - f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)." + f"Expected str or dict with 'type' field (text, image, image_url, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 71e686563a5..fcc909e5b64 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1114,3 +1114,70 @@ def test_count_content_list_rejects_unknown_type(): message = str(exc_info.value) assert "Invalid content item type: totally_unknown_block" in message assert "tool_reference" in message + + +def _png_base64(width: int, height: int) -> str: + import base64 + import struct + import zlib + + ihdr_data = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + ihdr = struct.pack(">I", 13) + b"IHDR" + ihdr_data + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr_data)) + return base64.b64encode(b"\x89PNG\r\n\x1a\n" + ihdr).decode() + + +def test_token_counter_anthropic_image_block_counts_instead_of_raising(): + """Anthropic-format image blocks ({"type": "image", "source": ...}) must + count instead of raising; a raise here breaks router pre-call checks and + cost tracking for anthropic messages traffic""" + from litellm.utils import token_counter + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _png_base64(1, 1), + }, + }, + ], + } + ] + text_only = token_counter( + model="gpt-4o", messages=[{"role": "user", "content": "What is in this image?"}] + ) + with_image = token_counter(model="gpt-4o", messages=messages) + assert with_image > text_only + + +def test_token_counter_anthropic_image_block_scales_with_dimensions(): + """Large images must count more than small ones; a flat estimate would + under-reserve budget for high-resolution images""" + from litellm.litellm_core_utils.token_counter import _count_anthropic_image_tokens + + small = _count_anthropic_image_tokens( + {"type": "base64", "media_type": "image/png", "data": _png_base64(1, 1)}, + use_default_image_token_count=False, + ) + large = _count_anthropic_image_tokens( + {"type": "base64", "media_type": "image/png", "data": _png_base64(2048, 1024)}, + use_default_image_token_count=False, + ) + assert small > 0 + assert large > small + + +def test_token_counter_anthropic_image_block_missing_source_uses_default(): + from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT + from litellm.litellm_core_utils.token_counter import _count_anthropic_image_tokens + + assert _count_anthropic_image_tokens(None, use_default_image_token_count=False) == DEFAULT_IMAGE_TOKEN_COUNT + assert ( + _count_anthropic_image_tokens({"type": "base64"}, use_default_image_token_count=False) + == DEFAULT_IMAGE_TOKEN_COUNT + )