Merge 228914b0bd into d392e7faae

litellm/litellm_core_utils/token_counter.py

@@ -695,6 +695,51 @@ def _count_anthropic_content(
     return tokens
 
 
+def _anthropic_image_source_to_data_uri(source: object) -> str | None:
+    """Convert an Anthropic image source to the URL / data-URI form ``calculate_img_tokens`` expects."""
+    if not isinstance(source, dict):
+        return None
+    source_type: Final = source.get("type")
+    if source_type == "url":
+        url: Final = source.get("url")
+        return url if isinstance(url, str) and url else None
+    if source_type == "base64":
+        raw_data: Final = source.get("data")
+        if not isinstance(raw_data, str) or not raw_data:
+            return None
+        media_type: Final = source.get("media_type") or "image/png"
+        return f"data:{media_type};base64,{raw_data}"
+    return None
+
+
+def _count_anthropic_image_tokens(
+    source: object,
+    use_default_image_token_count: bool,
+) -> int:
+    """
+    Count tokens for an Anthropic image content block source
+    ({"type": "base64", "media_type": ..., "data": ...} or {"type": "url", "url": ...}).
+
+    Converts the source to the URL / data-URI form understood by
+    _count_image_tokens so the dimension-aware estimate applies; falls back to
+    DEFAULT_IMAGE_TOKEN_COUNT when the source cannot be interpreted.
+    """
+    data: Final = _anthropic_image_source_to_data_uri(source)
+    if data is None:
+        return DEFAULT_IMAGE_TOKEN_COUNT
+    try:
+        # mode="high" applies the dimension-based tile math; "auto" resolves to
+        # the flat low-detail estimate, and anthropic image billing always
+        # scales with dimensions (there is no detail tier to select)
+        return calculate_img_tokens(
+            data=data,
+            mode="high",
+            use_default_image_token_count=use_default_image_token_count,
+        )
+    except (ValueError, TypeError, struct.error):
+        return DEFAULT_IMAGE_TOKEN_COUNT
+
+
 def _count_content_list(
     count_function: TokenCounterFunction,
     content_list: OpenAIMessageContent,
@@ -714,6 +759,12 @@ def _count_content_list(
             elif c["type"] == "image_url":
                 image_url = c.get("image_url")
                 num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
+            elif c["type"] == "image":
+                # Anthropic-format image block ({"type": "image", "source": ...}).
+                # Reuse the dimension-aware image_url counting by converting the
+                # source to a URL / data URI; a flat default would under-count
+                # large images since billed tokens scale with dimensions.
+                num_tokens += _count_anthropic_image_tokens(c.get("source"), use_default_image_token_count)
             elif c["type"] in ("tool_use", "tool_result"):
                 num_tokens += _count_anthropic_content(
                     c,
@@ -742,7 +793,7 @@ def _count_content_list(
                 content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
                 raise ValueError(
                     f"Invalid content item type: {content_type}. "
-                    f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)."
+                    f"Expected str or dict with 'type' field (text, image, image_url, tool_use, tool_result, thinking, tool_reference)."
                 )
         return num_tokens
     except Exception as e:

tests/test_litellm/litellm_core_utils/test_token_counter.py

@@ -1160,3 +1160,70 @@ def test_count_content_list_rejects_unknown_type():
     message = str(exc_info.value)
     assert "Invalid content item type: totally_unknown_block" in message
     assert "tool_reference" in message
+
+
+def _png_base64(width: int, height: int) -> str:
+    import base64
+    import struct
+    import zlib
+
+    ihdr_data = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
+    ihdr = struct.pack(">I", 13) + b"IHDR" + ihdr_data + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr_data))
+    return base64.b64encode(b"\x89PNG\r\n\x1a\n" + ihdr).decode()
+
+
+def test_token_counter_anthropic_image_block_counts_instead_of_raising():
+    """Anthropic-format image blocks ({"type": "image", "source": ...}) must
+    count instead of raising; a raise here breaks router pre-call checks and
+    cost tracking for anthropic messages traffic"""
+    from litellm.utils import token_counter
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": _png_base64(1, 1),
+                    },
+                },
+            ],
+        }
+    ]
+    text_only = token_counter(
+        model="gpt-4o", messages=[{"role": "user", "content": "What is in this image?"}]
+    )
+    with_image = token_counter(model="gpt-4o", messages=messages)
+    assert with_image > text_only
+
+
+def test_token_counter_anthropic_image_block_scales_with_dimensions():
+    """Large images must count more than small ones; a flat estimate would
+    under-reserve budget for high-resolution images"""
+    from litellm.litellm_core_utils.token_counter import _count_anthropic_image_tokens
+
+    small = _count_anthropic_image_tokens(
+        {"type": "base64", "media_type": "image/png", "data": _png_base64(1, 1)},
+        use_default_image_token_count=False,
+    )
+    large = _count_anthropic_image_tokens(
+        {"type": "base64", "media_type": "image/png", "data": _png_base64(2048, 1024)},
+        use_default_image_token_count=False,
+    )
+    assert small > 0
+    assert large > small
+
+
+def test_token_counter_anthropic_image_block_missing_source_uses_default():
+    from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
+    from litellm.litellm_core_utils.token_counter import _count_anthropic_image_tokens
+
+    assert _count_anthropic_image_tokens(None, use_default_image_token_count=False) == DEFAULT_IMAGE_TOKEN_COUNT
+    assert (
+        _count_anthropic_image_tokens({"type": "base64"}, use_default_image_token_count=False)
+        == DEFAULT_IMAGE_TOKEN_COUNT
+    )