diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 858b078d626..98bcedf43fe 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,7 +3,7 @@ import base64 import io import struct -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from typing import Any, Final, Literal, cast import tiktoken @@ -25,14 +25,21 @@ from litellm.litellm_core_utils.default_encoding import encoding as default_enco from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.types.llms.anthropic import ( + AnthropicContentParamSource, + AnthropicContentParamSourceFileId, + AnthropicContentParamSourceUrl, + AnthropicMessagesDocumentParam, + AnthropicMessagesImageParam, + AnthropicMessagesTextParam, AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, ) from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionDocumentObject, ChatCompletionNamedToolChoiceParam, ChatCompletionToolParam, - OpenAIMessageContent, + OpenAIMessageContentListBlock, ) from litellm.types.utils import Message, SelectTokenizerResponse @@ -346,7 +353,7 @@ def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, text: str | list[str] | None = None, - messages: list[AllMessageValues | Message] | None = None, + messages: Sequence[AllMessageValues | Message] | None = None, count_response_tokens: bool | None = False, tools: list[ChatCompletionToolParam] | None = None, tool_choice: ChatCompletionNamedToolChoiceParam | None = None, @@ -646,6 +653,57 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type: return expected_cls +def _anthropic_image_source_data( + source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId, +) -> str: + """ + Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices. + + Returns "" for a `file` source, whose bytes the proxy cannot resolve locally. + """ + if source["type"] == "base64": + data: Final = source.get("data") + if not data: + return "" + media_type: Final = source.get("media_type") or "image/png" + return f"data:{media_type};base64,{data}" + if source["type"] == "url": + return source.get("url") or "" + return "" + + +def _count_document_tokens( + document: ChatCompletionDocumentObject | AnthropicMessagesDocumentParam, + count_function: TokenCounterFunction, + use_default_image_token_count: bool, + default_token_count: int | None, +) -> int: + """ + Count an Anthropic `document` block: its title and context text, plus the source itself. + + Text-bearing sources (`text`, `content`) count their text; opaque ones (`base64`, `url`, + `file`) are priced like an image, since their bytes cannot be tokenized locally. + """ + source: Final = document["source"] + metadata_tokens: Final = sum( + count_function(text) for text in (document.get("title"), document.get("context")) if text + ) + if source["type"] == "text": + return metadata_tokens + count_function(source["data"]) + if source["type"] == "content": + content: Final = source["content"] + if isinstance(content, str): + return metadata_tokens + count_function(content) + return metadata_tokens + _count_content_list( + count_function, content, use_default_image_token_count, default_token_count + ) + return metadata_tokens + calculate_img_tokens( + data=_anthropic_image_source_data(source), + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + + def _count_anthropic_content( content: Mapping[str, Any], count_function: TokenCounterFunction, @@ -697,12 +755,22 @@ def _count_anthropic_content( def _count_content_list( count_function: TokenCounterFunction, - content_list: OpenAIMessageContent, + content_list: str + | Iterable[ + OpenAIMessageContentListBlock + | AnthropicMessagesTextParam + | AnthropicMessagesImageParam + | AnthropicMessagesDocumentParam + ], use_default_image_token_count: bool, default_token_count: int | None, ) -> int: """ Recursively count tokens from a list of content blocks. + + The block union is wider than OpenAI's: the proxy's Anthropic endpoints count + their native blocks through this same helper, so an `image` block is as much + an input here as OpenAI's `image_url`. """ try: num_tokens = 0 @@ -714,6 +782,19 @@ def _count_content_list( elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) + elif c["type"] == "image": + num_tokens += calculate_img_tokens( + data=_anthropic_image_source_data(c["source"]), + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + elif c["type"] == "document": + num_tokens += _count_document_tokens( + c, + count_function, + use_default_image_token_count, + default_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -742,7 +823,8 @@ def _count_content_list( content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__ raise ValueError( f"Invalid content item type: {content_type}. " - f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)." + f"Expected str or dict with 'type' field " + f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af26a9f669e..e1065638217 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -665,7 +665,12 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseUsageBlock, ) -from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionSystemMessage, + ChatCompletionToolParam, + HttpxBinaryResponseContent, +) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, @@ -12094,6 +12099,13 @@ async def _try_provider_token_count( return result +def _system_message(system: object) -> ChatCompletionSystemMessage | None: + if not isinstance(system, (str, list)) or not system: + return None + message: Final[ChatCompletionSystemMessage] = {"role": "system", "content": system} + return message + + @router.post( "/utils/token_counter", tags=["llm utils"], @@ -12192,10 +12204,21 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) _tokenizer_used: Final = litellm.utils._select_tokenizer(model=model_to_use, custom_tokenizer=custom_tokenizer) tokenizer_used: Final = str(_tokenizer_used["type"]) + system_message: Final = _system_message(system) + typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes + Sequence[AllMessageValues] | None, messages + ) + counted_messages: Final = ( + typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) + ) + counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats + list[ChatCompletionToolParam] | None, tools + ) total_tokens: Final = await asyncify(litellm.token_counter)( model=model_to_use, text=prompt, - messages=messages, + messages=counted_messages, + tools=counted_tools, custom_tokenizer=_tokenizer_used, ) return TokenCountResponse( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 7805dd595a2..b3462203c4b 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from enum import Enum from typing import Any, Final, Literal, TypeAlias @@ -254,6 +254,17 @@ class AnthropicContentParamSourceFileId(TypedDict): file_id: str +class AnthropicContentParamSourceText(TypedDict): + type: ReadOnly[Literal["text"]] + media_type: ReadOnly[Literal["text/plain"]] + data: ReadOnly[str] + + +class AnthropicContentParamSourceContent(TypedDict): + type: ReadOnly[Literal["content"]] + content: ReadOnly[str | Sequence["AnthropicMessagesTextParam | AnthropicMessagesImageParam"]] + + class AnthropicMessagesContainerUploadParam(TypedDict, total=False): type: Required[Literal["container_upload"]] file_id: str @@ -305,7 +316,13 @@ AnthropicCitation = AnthropicCitationPageLocation | AnthropicCitationCharLocatio class AnthropicMessagesDocumentParam(TypedDict, total=False): type: Required[Literal["document"]] - source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl] + source: Required[ + AnthropicContentParamSource + | AnthropicContentParamSourceFileId + | AnthropicContentParamSourceUrl + | AnthropicContentParamSourceText + | AnthropicContentParamSourceContent + ] cache_control: dict | ChatCompletionCachedContent | None title: str context: str diff --git a/litellm/utils.py b/litellm/utils.py index b75d0161cb4..ff2d9b3a78d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2302,7 +2302,7 @@ def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, text: str | list[str] | None = None, - messages: list | None = None, + messages: Sequence | None = None, count_response_tokens: bool | None = False, tools: list[ChatCompletionToolParam] | None = None, tool_choice: ChatCompletionNamedToolChoiceParam | None = None, @@ -7740,7 +7740,7 @@ def convert_to_dict(message: BaseModel | dict) -> dict: raise TypeError(f"Invalid message type: {type(message)}. Expected dict or Pydantic model.") -def convert_list_message_to_dict(messages: list): +def convert_list_message_to_dict(messages: Sequence): new_messages: Final = [] for message in messages: convert_msg_to_dict = cast(AllMessageValues, convert_to_dict(message)) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a2590dbca2d..9a10769758e 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1160,3 +1160,232 @@ def test_count_content_list_rejects_unknown_type(): message = str(exc_info.value) assert "Invalid content item type: totally_unknown_block" in message assert "tool_reference" in message + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}, + {"type": "url", "url": "https://example.com/image.png"}, + {"type": "file", "file_id": "file-abc123"}, + ], + ids=["base64", "url", "file"], +) +def test_token_counter_with_anthropic_image_block(source: dict[str, str]): + """ + Anthropic-native `image` blocks must NOT raise, for every source variant. + + Before this fix `_count_content_list` raised + `Invalid content item type: image`. That 500s /v1/messages/count_tokens and + /utils/token_counter, and it makes the router's context-window pre-call + check swallow the error and return every deployment unfiltered, so an + oversized prompt carrying an image is dispatched upstream instead of being + rejected locally. + """ + from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image", "source": source}, + ], + } + ] + + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + use_default_image_token_count=True, + ) + assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, ( + f"Expected the image block to contribute tokens, got {tokens}" + ) + + +def test_anthropic_image_block_matches_equivalent_image_url(): + """ + An Anthropic `image` block must price identically to the OpenAI `image_url` + block carrying the same bytes, so the count does not depend on which + endpoint shape the caller used. + """ + anthropic_messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + ], + } + ] + openai_messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + } + ], + } + ] + + anthropic_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages + ) + openai_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages + ) + assert anthropic_tokens == openai_tokens + + +def test_anthropic_image_block_nested_in_tool_result(): + """ + An `image` block nested inside a `tool_result.content` list must be counted + too. `_count_anthropic_content` recurses back into `_count_content_list`, so + the nested case failed for the same reason the top-level one did. + """ + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + ], + } + ], + } + ] + + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + use_default_image_token_count=True, + ) + assert tokens > 0 + + +def test_anthropic_image_block_with_empty_base64_data(): + """ + A base64 source carrying no bytes must still price as an image rather than + raise: the block is well-formed enough to count, and an empty `data` only + means there is nothing to measure the dimensions from. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + tokens = _count_content_list( + count_function=len, + content_list=[ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}} + ], + use_default_image_token_count=False, + default_token_count=None, + ) + assert tokens > 0 + + +def test_anthropic_image_block_without_source_raises(): + """ + An `image` block with no `source` is malformed, and must fail the same way + the OpenAI `image_url` block with no `url` does - a ValueError the caller + can turn into a 400 - instead of being silently counted as a valid image. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + with pytest.raises(ValueError, match="Error getting number of tokens from content list"): + _count_content_list( + count_function=len, + content_list=[{"type": "image"}], + use_default_image_token_count=False, + default_token_count=None, + ) + + # ... and `default_token_count`, the caller's opt-out from raising, still wins. + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "image"}], + use_default_image_token_count=False, + default_token_count=7, + ) + == 7 + ) + + +def _count_user_content(content: list[dict]) -> int: + from litellm.litellm_core_utils.token_counter import token_counter + + return token_counter( + model="anthropic/claude-fable-5", + messages=[{"role": "user", "content": content}], + use_default_image_token_count=True, + ) + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + {"type": "url", "url": "https://example.com/report.pdf"}, + {"type": "file", "file_id": "file-abc123"}, + ], + ids=["base64", "url", "file"], +) +def test_anthropic_document_block_with_opaque_source_is_priced_like_an_image(source: dict[str, str]): + """ + A `document` whose bytes cannot be tokenized locally must not raise (it 500ed + /v1/messages/count_tokens before) and is priced exactly like an `image` block. + """ + prompt = {"type": "text", "text": "Summarize this file."} + + assert _count_user_content([prompt, {"type": "document", "source": source}]) == _count_user_content( + [prompt, {"type": "image", "source": source}] + ) + + +def test_anthropic_document_block_text_sources_count_their_text(): + """`text` and `content` document sources count the text they carry, as inline text blocks would.""" + prompt = {"type": "text", "text": "Summarize this file."} + body = {"type": "text", "text": "Revenue grew eleven percent while churn fell to two percent."} + picture = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}} + + text_source = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": body["text"]}} + assert _count_user_content([prompt, text_source]) == _count_user_content([prompt, body]) + + string_content = {"type": "document", "source": {"type": "content", "content": body["text"]}} + assert _count_user_content([prompt, string_content]) == _count_user_content([prompt, body]) + + block_content = {"type": "document", "source": {"type": "content", "content": [body, picture]}} + assert _count_user_content([prompt, block_content]) == _count_user_content([prompt, body, picture]) + + +def test_anthropic_document_title_and_context_add_their_tokens(): + prompt = {"type": "text", "text": "Summarize this file."} + source = {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"} + described = {"type": "document", "source": source, "title": "Q3 board packet", "context": "Shared by finance"} + + assert _count_user_content([prompt, described]) == _count_user_content( + [ + prompt, + {"type": "text", "text": "Q3 board packet"}, + {"type": "text", "text": "Shared by finance"}, + {"type": "document", "source": source}, + ] + ) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py index f39192b171b..6fffead102f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py @@ -184,3 +184,48 @@ def test_transform_request_unsafe_body(client, auth_as, monkeypatch): response = client.post("/utils/transform_request", json=payload) assert response.status_code == 400 assert "unsafe" in response.text or "error" in response.text + + +def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client, auth_as, monkeypatch): + """ + Without a provider counter the route falls back to ``litellm.token_counter``. That count + must include the request's tools and system prompt, and Anthropic ``image`` and ``document`` + blocks must be counted instead of turning the whole request into a 500. + """ + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False) + system = [{"type": "text", "text": "You are a terse assistant. Answer in one sentence."}] + tools = [ + { + "name": "get_weather", + "description": "Look up the current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ] + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this file?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}, + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}}, + ], + } + ] + + def count(payload: dict) -> int: + with auth_as(): + response = client.post("/utils/token_counter", json={"model": "claude-fable-5", **payload}) + assert response.status_code == 200, response.text + return response.json()["total_tokens"] + + bare = count({"messages": messages}) + full = count({"messages": messages, "tools": tools, "system": system}) + + assert bare == litellm.token_counter(model="claude-fable-5", messages=messages) + assert full == litellm.token_counter( + model="claude-fable-5", + messages=[{"role": "system", "content": system}, *messages], + tools=tools, + ) + assert full > bare