diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 858b078d626..f101188a692 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,7 +3,7 @@ import base64 import io import struct -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from typing import Any, Final, Literal, cast import tiktoken @@ -25,6 +25,10 @@ from litellm.litellm_core_utils.default_encoding import encoding as default_enco from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.types.llms.anthropic import ( + AnthropicContentParamSource, + AnthropicContentParamSourceFileId, + AnthropicContentParamSourceUrl, + AnthropicMessagesImageParam, AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, ) @@ -32,7 +36,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionNamedToolChoiceParam, ChatCompletionToolParam, - OpenAIMessageContent, + OpenAIMessageContentListBlock, ) from litellm.types.utils import Message, SelectTokenizerResponse @@ -646,6 +650,25 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type: return expected_cls +def _anthropic_image_source_data( + source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId, +) -> str: + """ + Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices. + + Returns "" for a `file` source, whose bytes the proxy cannot resolve locally. + """ + if source["type"] == "base64": + data: Final = source.get("data") + if not data: + return "" + media_type: Final = source.get("media_type") or "image/png" + return f"data:{media_type};base64,{data}" + if source["type"] == "url": + return source.get("url") or "" + return "" + + def _count_anthropic_content( content: Mapping[str, Any], count_function: TokenCounterFunction, @@ -697,12 +720,16 @@ def _count_anthropic_content( def _count_content_list( count_function: TokenCounterFunction, - content_list: OpenAIMessageContent, + content_list: str | Iterable[OpenAIMessageContentListBlock | AnthropicMessagesImageParam], use_default_image_token_count: bool, default_token_count: int | None, ) -> int: """ Recursively count tokens from a list of content blocks. + + The block union is wider than OpenAI's: the proxy's Anthropic endpoints count + their native blocks through this same helper, so an `image` block is as much + an input here as OpenAI's `image_url`. """ try: num_tokens = 0 @@ -714,6 +741,12 @@ def _count_content_list( elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) + elif c["type"] == "image": + num_tokens += calculate_img_tokens( + data=_anthropic_image_source_data(c["source"]), + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -742,7 +775,8 @@ def _count_content_list( content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__ raise ValueError( f"Invalid content item type: {content_type}. " - f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)." + f"Expected str or dict with 'type' field " + f"(text, image_url, image, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a2590dbca2d..7019577cca0 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1160,3 +1160,170 @@ def test_count_content_list_rejects_unknown_type(): message = str(exc_info.value) assert "Invalid content item type: totally_unknown_block" in message assert "tool_reference" in message + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}, + {"type": "url", "url": "https://example.com/image.png"}, + {"type": "file", "file_id": "file-abc123"}, + ], + ids=["base64", "url", "file"], +) +def test_token_counter_with_anthropic_image_block(source: dict[str, str]): + """ + Anthropic-native `image` blocks must NOT raise, for every source variant. + + Before this fix `_count_content_list` raised + `Invalid content item type: image`. That 500s /v1/messages/count_tokens and + /utils/token_counter, and it makes the router's context-window pre-call + check swallow the error and return every deployment unfiltered, so an + oversized prompt carrying an image is dispatched upstream instead of being + rejected locally. + """ + from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image", "source": source}, + ], + } + ] + + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + use_default_image_token_count=True, + ) + assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, ( + f"Expected the image block to contribute tokens, got {tokens}" + ) + + +def test_anthropic_image_block_matches_equivalent_image_url(): + """ + An Anthropic `image` block must price identically to the OpenAI `image_url` + block carrying the same bytes, so the count does not depend on which + endpoint shape the caller used. + """ + anthropic_messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + ], + } + ] + openai_messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + } + ], + } + ] + + anthropic_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages + ) + openai_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages + ) + assert anthropic_tokens == openai_tokens + + +def test_anthropic_image_block_nested_in_tool_result(): + """ + An `image` block nested inside a `tool_result.content` list must be counted + too. `_count_anthropic_content` recurses back into `_count_content_list`, so + the nested case failed for the same reason the top-level one did. + """ + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + ], + } + ], + } + ] + + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + use_default_image_token_count=True, + ) + assert tokens > 0 + + +def test_anthropic_image_block_with_empty_base64_data(): + """ + A base64 source carrying no bytes must still price as an image rather than + raise: the block is well-formed enough to count, and an empty `data` only + means there is nothing to measure the dimensions from. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + tokens = _count_content_list( + count_function=len, + content_list=[ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}} + ], + use_default_image_token_count=False, + default_token_count=None, + ) + assert tokens > 0 + + +def test_anthropic_image_block_without_source_raises(): + """ + An `image` block with no `source` is malformed, and must fail the same way + the OpenAI `image_url` block with no `url` does - a ValueError the caller + can turn into a 400 - instead of being silently counted as a valid image. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + with pytest.raises(ValueError): + _count_content_list( + count_function=len, + content_list=[{"type": "image"}], + use_default_image_token_count=False, + default_token_count=None, + ) + + # ... and `default_token_count`, the caller's opt-out from raising, still wins. + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "image"}], + use_default_image_token_count=False, + default_token_count=7, + ) + == 7 + )