diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e56dd83a277..2795fd81838 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -17,6 +17,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from openai.types.chat import ChatCompletionContentPartRefusalParam from pydantic import TypeAdapter from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar @@ -1812,7 +1813,13 @@ def _count_anthropic_content( _TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType( - {"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"} + { + "text": "text", + "refusal": "refusal", + "thinking": "thinking", + "redacted_thinking": "thinking", + "compaction": "content", + } ) @@ -1821,6 +1828,7 @@ def _count_content_list( content_list: str | Iterable[ OpenAIMessageContentListBlock + | ChatCompletionContentPartRefusalParam | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | AnthropicMessagesTextParam @@ -1891,7 +1899,7 @@ def _count_content_list( raise UnsupportedContentTypeError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " - f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " + f"(text, refusal, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " f"compaction, tool_reference, search_result)." ) return num_tokens diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d0b837646b4..264d71de332 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -21,6 +21,7 @@ from unittest.mock import MagicMock, sentinel import anyio.to_thread import pytest import tiktoken +from openai.types.chat import ChatCompletionContentPartRefusalParam from PIL import Image from tokenizers import Regex, Tokenizer, models, pre_tokenizers @@ -37,6 +38,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( + _count_content_list, _encoding_count, _get_exact_count_function, _get_extrapolating_count_function, @@ -49,7 +51,7 @@ from litellm.litellm_core_utils.token_counter import ( ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionTextObject, ChatCompletionToolParam from litellm.types.utils import SelectTokenizerResponse from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( @@ -93,6 +95,28 @@ def test_token_counter_basic(): ) +@pytest.mark.parametrize("refusal_text", ["I cannot help with that request.", "", "抱歉,我无法帮助处理该请求。"]) +@pytest.mark.parametrize("default_token_count", [None, 0]) +def test_content_list_counts_refusal_text(refusal_text: str, default_token_count: int | None) -> None: + encoding: Final = tiktoken.get_encoding("cl100k_base") + count_function: Final[Callable[[str], int]] = lambda text: _encoding_count(encoding, text) + content: Final[list[ChatCompletionTextObject | ChatCompletionContentPartRefusalParam]] = [ + {"type": "text", "text": "Safety notice."}, + {"type": "refusal", "refusal": refusal_text}, + ] + expected: Final = count_function("Safety notice.") + count_function(refusal_text) + + assert ( + _count_content_list( + count_function=count_function, + content_list=content, + use_default_image_token_count=False, + default_token_count=default_token_count, + ) + == expected + ) + + def test_token_counter_large_repeated_text_is_encoded_in_bounded_chunks(): text_length: Final = 1024 * 1024 messages: Final = [{"role": "user", "content": [{"type": "text", "text": "A" * text_length}]}]