Merge remote-tracking branch 'origin/pr-36671-head' into litellm_count_tokens_fallback_tools_system

litellm/litellm_core_utils/token_counter.py

@@ -3,7 +3,7 @@
 import base64
 import io
 import struct
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from typing import Any, Final, Literal, cast
 
 import tiktoken
@@ -25,6 +25,10 @@ from litellm.litellm_core_utils.default_encoding import encoding as default_enco
 from litellm.litellm_core_utils.url_utils import safe_get
 from litellm.llms.custom_httpx.http_handler import _get_httpx_client
 from litellm.types.llms.anthropic import (
+    AnthropicContentParamSource,
+    AnthropicContentParamSourceFileId,
+    AnthropicContentParamSourceUrl,
+    AnthropicMessagesImageParam,
     AnthropicMessagesToolResultParam,
     AnthropicMessagesToolUseParam,
 )
@@ -32,7 +36,7 @@ from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionNamedToolChoiceParam,
     ChatCompletionToolParam,
-    OpenAIMessageContent,
+    OpenAIMessageContentListBlock,
 )
 from litellm.types.utils import Message, SelectTokenizerResponse
 
@@ -646,6 +650,25 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type:
     return expected_cls
 
 
+def _anthropic_image_source_data(
+    source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId,
+) -> str:
+    """
+    Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices.
+
+    Returns "" for a `file` source, whose bytes the proxy cannot resolve locally.
+    """
+    if source["type"] == "base64":
+        data: Final = source.get("data")
+        if not data:
+            return ""
+        media_type: Final = source.get("media_type") or "image/png"
+        return f"data:{media_type};base64,{data}"
+    if source["type"] == "url":
+        return source.get("url") or ""
+    return ""
+
+
 def _count_anthropic_content(
     content: Mapping[str, Any],
     count_function: TokenCounterFunction,
@@ -697,12 +720,16 @@ def _count_anthropic_content(
 
 def _count_content_list(
     count_function: TokenCounterFunction,
-    content_list: OpenAIMessageContent,
+    content_list: str | Iterable[OpenAIMessageContentListBlock | AnthropicMessagesImageParam],
     use_default_image_token_count: bool,
     default_token_count: int | None,
 ) -> int:
     """
     Recursively count tokens from a list of content blocks.
+
+    The block union is wider than OpenAI's: the proxy's Anthropic endpoints count
+    their native blocks through this same helper, so an `image` block is as much
+    an input here as OpenAI's `image_url`.
     """
     try:
         num_tokens = 0
@@ -714,6 +741,12 @@ def _count_content_list(
             elif c["type"] == "image_url":
                 image_url = c.get("image_url")
                 num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
+            elif c["type"] == "image":
+                num_tokens += calculate_img_tokens(
+                    data=_anthropic_image_source_data(c["source"]),
+                    mode="auto",
+                    use_default_image_token_count=use_default_image_token_count,
+                )
             elif c["type"] in ("tool_use", "tool_result"):
                 num_tokens += _count_anthropic_content(
                     c,
@@ -742,7 +775,8 @@ def _count_content_list(
                 content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
                 raise ValueError(
                     f"Invalid content item type: {content_type}. "
-                    f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)."
+                    f"Expected str or dict with 'type' field "
+                    f"(text, image_url, image, tool_use, tool_result, thinking, tool_reference)."
                 )
         return num_tokens
     except Exception as e:

tests/test_litellm/litellm_core_utils/test_token_counter.py

@@ -1160,3 +1160,170 @@ def test_count_content_list_rejects_unknown_type():
     message = str(exc_info.value)
     assert "Invalid content item type: totally_unknown_block" in message
     assert "tool_reference" in message
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="},
+        {"type": "url", "url": "https://example.com/image.png"},
+        {"type": "file", "file_id": "file-abc123"},
+    ],
+    ids=["base64", "url", "file"],
+)
+def test_token_counter_with_anthropic_image_block(source: dict[str, str]):
+    """
+    Anthropic-native `image` blocks must NOT raise, for every source variant.
+
+    Before this fix `_count_content_list` raised
+    `Invalid content item type: image`. That 500s /v1/messages/count_tokens and
+    /utils/token_counter, and it makes the router's context-window pre-call
+    check swallow the error and return every deployment unfiltered, so an
+    oversized prompt carrying an image is dispatched upstream instead of being
+    rejected locally.
+    """
+    from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {"type": "image", "source": source},
+            ],
+        }
+    ]
+
+    tokens = token_counter(
+        model="anthropic/claude-sonnet-4-5-20250929",
+        messages=messages,
+        use_default_image_token_count=True,
+    )
+    assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, (
+        f"Expected the image block to contribute tokens, got {tokens}"
+    )
+
+
+def test_anthropic_image_block_matches_equivalent_image_url():
+    """
+    An Anthropic `image` block must price identically to the OpenAI `image_url`
+    block carrying the same bytes, so the count does not depend on which
+    endpoint shape the caller used.
+    """
+    anthropic_messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": "iVBORw0KGgo=",
+                    },
+                }
+            ],
+        }
+    ]
+    openai_messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="},
+                }
+            ],
+        }
+    ]
+
+    anthropic_tokens = token_counter(
+        model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages
+    )
+    openai_tokens = token_counter(
+        model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages
+    )
+    assert anthropic_tokens == openai_tokens
+
+
+def test_anthropic_image_block_nested_in_tool_result():
+    """
+    An `image` block nested inside a `tool_result.content` list must be counted
+    too. `_count_anthropic_content` recurses back into `_count_content_list`, so
+    the nested case failed for the same reason the top-level one did.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "tool_result",
+                    "tool_use_id": "toolu_01",
+                    "content": [
+                        {
+                            "type": "image",
+                            "source": {
+                                "type": "base64",
+                                "media_type": "image/png",
+                                "data": "iVBORw0KGgo=",
+                            },
+                        }
+                    ],
+                }
+            ],
+        }
+    ]
+
+    tokens = token_counter(
+        model="anthropic/claude-sonnet-4-5-20250929",
+        messages=messages,
+        use_default_image_token_count=True,
+    )
+    assert tokens > 0
+
+
+def test_anthropic_image_block_with_empty_base64_data():
+    """
+    A base64 source carrying no bytes must still price as an image rather than
+    raise: the block is well-formed enough to count, and an empty `data` only
+    means there is nothing to measure the dimensions from.
+    """
+    from litellm.litellm_core_utils.token_counter import _count_content_list
+
+    tokens = _count_content_list(
+        count_function=len,
+        content_list=[
+            {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}
+        ],
+        use_default_image_token_count=False,
+        default_token_count=None,
+    )
+    assert tokens > 0
+
+
+def test_anthropic_image_block_without_source_raises():
+    """
+    An `image` block with no `source` is malformed, and must fail the same way
+    the OpenAI `image_url` block with no `url` does - a ValueError the caller
+    can turn into a 400 - instead of being silently counted as a valid image.
+    """
+    from litellm.litellm_core_utils.token_counter import _count_content_list
+
+    with pytest.raises(ValueError):
+        _count_content_list(
+            count_function=len,
+            content_list=[{"type": "image"}],
+            use_default_image_token_count=False,
+            default_token_count=None,
+        )
+
+    # ... and `default_token_count`, the caller's opt-out from raising, still wins.
+    assert (
+        _count_content_list(
+            count_function=len,
+            content_list=[{"type": "image"}],
+            use_default_image_token_count=False,
+            default_token_count=7,
+        )
+        == 7
+    )