mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(token-counter): count Anthropic search_result blocks instead of failing
POST /v1/messages/count_tokens answered 500 for a message containing a search_result content block, because the local token counter only knew text, image, document, file, tool and thinking blocks. Count a search_result block's source, title and text content through the existing Anthropic TypedDict-driven counter, so it is counted like the other text-bearing blocks. Fixes #47952
This commit is contained in:
parent
f2118fc767
commit
dc63bac3d0
3 changed files with 31 additions and 2 deletions
|
|
@ -55,6 +55,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesTextParam,
|
||||
AnthropicMessagesToolResultParam,
|
||||
AnthropicMessagesToolUseParam,
|
||||
AnthropicSearchResultParam,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -825,6 +826,7 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
|
|||
mapping: Final = {
|
||||
"tool_use": AnthropicMessagesToolUseParam,
|
||||
"tool_result": AnthropicMessagesToolResultParam,
|
||||
"search_result": AnthropicSearchResultParam,
|
||||
}
|
||||
|
||||
expected_cls: Final = mapping.get(content_type) if isinstance(content_type, str) else None
|
||||
|
|
@ -1028,7 +1030,7 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result"):
|
||||
elif c["type"] in ("tool_use", "tool_result", "search_result"):
|
||||
num_tokens += _count_anthropic_content(
|
||||
c,
|
||||
count_function,
|
||||
|
|
@ -1058,7 +1060,7 @@ def _count_content_list(
|
|||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"tool_reference)."
|
||||
f"tool_reference, search_result)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -209,6 +209,13 @@ class AnthropicMessagesTextParam(TypedDict, total=False):
|
|||
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
|
||||
|
||||
|
||||
class AnthropicSearchResultParam(TypedDict, total=False):
|
||||
type: Required[ReadOnly[Literal["search_result"]]]
|
||||
source: Required[ReadOnly[str]]
|
||||
title: Required[ReadOnly[str]]
|
||||
content: Required[ReadOnly[Iterable[AnthropicMessagesTextParam]]]
|
||||
|
||||
|
||||
class AnthropicMessagesToolUseParam(TypedDict, total=False):
|
||||
type: Required[Literal["tool_use"]]
|
||||
id: str
|
||||
|
|
|
|||
|
|
@ -1557,6 +1557,26 @@ def test_token_counter_with_thinking_content():
|
|||
), f"Expected minimal token count for empty thinking block, got {tokens_no_thinking}"
|
||||
|
||||
|
||||
def test_token_counter_counts_anthropic_search_result_text() -> None:
|
||||
search_result: Final = {
|
||||
"type": "search_result",
|
||||
"source": "https://example.com/paris",
|
||||
"title": "Paris",
|
||||
"content": [{"type": "text", "text": "Paris is the capital of France."}],
|
||||
"citations": {"enabled": True},
|
||||
}
|
||||
as_text_blocks: Final = [
|
||||
{"type": "text", "text": "https://example.com/paris"},
|
||||
{"type": "text", "text": "Paris"},
|
||||
{"type": "text", "text": "Paris is the capital of France."},
|
||||
]
|
||||
|
||||
def count(content: list[dict[str, object]]) -> int:
|
||||
return token_counter(model="gpt-4o", messages=[{"role": "user", "content": content}])
|
||||
|
||||
assert count([search_result]) == count(as_text_blocks)
|
||||
|
||||
|
||||
def test_token_counter_with_redacted_thinking_content():
|
||||
"""
|
||||
A replayed redacted_thinking block (Anthropic redacted reasoning, or the /v1/messages bridge's stand-in
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue