chore(token_counter): drop docstrings and test prose that restated the count_tokens branches

litellm/litellm_core_utils/token_counter.py

@@ -656,11 +656,6 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type:
 def _anthropic_image_source_data(
     source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId,
 ) -> str:
-    """
-    Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices.
-
-    Returns "" for a `file` source, whose bytes the proxy cannot resolve locally.
-    """
     if source["type"] == "base64":
         data: Final = source.get("data")
         if not data:
@@ -678,12 +673,6 @@ def _count_document_tokens(
     use_default_image_token_count: bool,
     default_token_count: int | None,
 ) -> int:
-    """
-    Count an Anthropic `document` block: its title and context text, plus the source itself.
-
-    Text-bearing sources (`text`, `content`) count their text; opaque ones (`base64`, `url`,
-    `file`) are priced like an image, since their bytes cannot be tokenized locally.
-    """
     source: Final = document["source"]
     metadata_tokens: Final = sum(
         count_function(text) for text in (document.get("title"), document.get("context")) if text
@@ -765,13 +754,7 @@ def _count_content_list(
     use_default_image_token_count: bool,
     default_token_count: int | None,
 ) -> int:
-    """
-    Recursively count tokens from a list of content blocks.
-
-    The block union is wider than OpenAI's: the proxy's Anthropic endpoints count
-    their native blocks through this same helper, so an `image` block is as much
-    an input here as OpenAI's `image_url`.
-    """
+    """Recursively count tokens from a list of content blocks."""
     try:
         num_tokens = 0
         for c in content_list:

tests/test_litellm/litellm_core_utils/test_token_counter.py

@@ -1172,16 +1172,7 @@ def test_count_content_list_rejects_unknown_type():
     ids=["base64", "url", "file"],
 )
 def test_token_counter_with_anthropic_image_block(source: dict[str, str]):
-    """
-    Anthropic-native `image` blocks must NOT raise, for every source variant.
-
-    Before this fix `_count_content_list` raised
-    `Invalid content item type: image`. That 500s /v1/messages/count_tokens and
-    /utils/token_counter, and it makes the router's context-window pre-call
-    check swallow the error and return every deployment unfiltered, so an
-    oversized prompt carrying an image is dispatched upstream instead of being
-    rejected locally.
-    """
+    """Anthropic `image` blocks must count for every source variant, not raise `Invalid content item type` (which the router's context-window pre-call check swallows into an unfiltered dispatch)."""
     from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT
 
     messages = [
@@ -1205,11 +1196,7 @@ def test_token_counter_with_anthropic_image_block(source: dict[str, str]):
 
 
 def test_anthropic_image_block_matches_equivalent_image_url():
-    """
-    An Anthropic `image` block must price identically to the OpenAI `image_url`
-    block carrying the same bytes, so the count does not depend on which
-    endpoint shape the caller used.
-    """
+    """An Anthropic `image` block prices identically to the OpenAI `image_url` carrying the same bytes."""
     anthropic_messages = [
         {
             "role": "user",
@@ -1247,11 +1234,7 @@ def test_anthropic_image_block_matches_equivalent_image_url():
 
 
 def test_anthropic_image_block_nested_in_tool_result():
-    """
-    An `image` block nested inside a `tool_result.content` list must be counted
-    too. `_count_anthropic_content` recurses back into `_count_content_list`, so
-    the nested case failed for the same reason the top-level one did.
-    """
+    """An `image` block nested in a `tool_result.content` list is counted through the same recursion."""
     messages = [
         {
             "role": "user",
@@ -1292,22 +1275,14 @@ def test_anthropic_image_block_nested_in_tool_result():
     ids=["base64", "url", "file"],
 )
 def test_anthropic_image_source_resolves_to_what_the_image_pricer_reads(source: dict[str, str], expected: str):
-    """
-    The image pricer reads either a data URI or a fetchable URL: a base64 source keeps its
-    media type inside the URI, a url source passes through untouched, and a file source has
-    no bytes the proxy can measure locally.
-    """
+    """base64 sources become a data URI, url sources pass through, file sources resolve to an empty string."""
     from litellm.litellm_core_utils.token_counter import _anthropic_image_source_data
 
     assert _anthropic_image_source_data(source) == expected
 
 
 def test_anthropic_image_block_with_empty_base64_data():
-    """
-    A base64 source carrying no bytes must still price as an image rather than
-    raise: the block is well-formed enough to count, and an empty `data` only
-    means there is nothing to measure the dimensions from.
-    """
+    """A base64 source with empty `data` prices as an image rather than raising."""
     from litellm.litellm_core_utils.token_counter import _count_content_list
 
     tokens = _count_content_list(
@@ -1322,11 +1297,7 @@ def test_anthropic_image_block_with_empty_base64_data():
 
 
 def test_anthropic_image_block_without_source_raises():
-    """
-    An `image` block with no `source` is malformed, and must fail the same way
-    the OpenAI `image_url` block with no `url` does - a ValueError the caller
-    can turn into a 400 - instead of being silently counted as a valid image.
-    """
+    """An `image` block with no `source` raises, matching the OpenAI `image_url`-without-`url` behavior."""
     from litellm.litellm_core_utils.token_counter import _count_content_list
 
     with pytest.raises(ValueError, match="Error getting number of tokens from content list"):
@@ -1369,10 +1340,7 @@ def _count_user_content(content: list[dict]) -> int:
     ids=["base64", "url", "file"],
 )
 def test_anthropic_document_block_with_opaque_source_is_priced_like_an_image(source: dict[str, str]):
-    """
-    A `document` whose bytes cannot be tokenized locally must not raise (it 500ed
-    /v1/messages/count_tokens before) and is priced exactly like an `image` block.
-    """
+    """A `document` whose bytes can't be tokenized locally is priced like an `image`, not raised on."""
     prompt = {"type": "text", "text": "Summarize this file."}
 
     assert _count_user_content([prompt, {"type": "document", "source": source}]) == _count_user_content(

tests/test_litellm/proxy/proxy_server/test_routes_utils.py

@@ -187,11 +187,7 @@ def test_transform_request_unsafe_body(client, auth_as, monkeypatch):
 
 
 def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client, auth_as, monkeypatch):
-    """
-    Without a provider counter the route falls back to ``litellm.token_counter``. That count
-    must include the request's tools and system prompt, and Anthropic ``image`` and ``document``
-    blocks must be counted instead of turning the whole request into a 500.
-    """
+    """The ``litellm.token_counter`` fallback counts the request's tools and system prompt, and Anthropic ``image``/``document`` blocks, instead of 500ing."""
     monkeypatch.setattr(proxy_server, "llm_router", None)
     monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False)
     system = [{"type": "text", "text": "You are a terse assistant. Answer in one sentence."}]
@@ -232,12 +228,7 @@ def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client,
 
 
 def test_token_counter_fallback_prompt_with_tools_does_not_500(client, auth_as, monkeypatch):
-    """
-    Regression: a raw-text ``prompt`` request that also carries ``tools`` (no ``messages``) must
-    still count. ``litellm.token_counter`` rejects tools on the text path, so the fallback route
-    only attaches tools when it is counting messages; otherwise this 500'd instead of returning
-    the plain text count.
-    """
+    """Regression: a ``prompt`` request carrying ``tools`` but no ``messages`` still counts, because the fallback attaches tools only when counting messages (``token_counter`` rejects tools on the text path)."""
     monkeypatch.setattr(proxy_server, "llm_router", None)
     monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False)
     prompt = "count the tokens in this sentence please"