mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-30 01:52:18 +00:00
fix(proxy): convert Anthropic tools to OpenAI format in token_counter fallback path
Fixes #26436 When /v1/messages/count_tokens falls back to local tokenizer, tools were passed in Anthropic format (name/description/input_schema) but _format_function_definitions() expects OpenAI format (type/function/parameters), causing AttributeError and tools being ignored. Added conversion of Anthropic-style tools to OpenAI format before calling token_counter() in the fallback path.
This commit is contained in:
parent
5bd59b33e6
commit
fb4d866992
3 changed files with 54 additions and 1 deletions
|
|
@ -509,7 +509,23 @@ def _count_extra(
|
|||
num_tokens = 3 # every reply is primed with <|start|>assistant<|message|>
|
||||
|
||||
if tools:
|
||||
num_tokens += count_function(_format_function_definitions(tools))
|
||||
# Normalize Anthropic-format tools to OpenAI format before counting
|
||||
normalized_tools = []
|
||||
for t in tools:
|
||||
if isinstance(t, dict) and "function" not in t and "input_schema" in t:
|
||||
normalized_tools.append(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": t.get("name"),
|
||||
"description": t.get("description", ""),
|
||||
"parameters": t.get("input_schema", {}),
|
||||
},
|
||||
}
|
||||
)
|
||||
else:
|
||||
normalized_tools.append(t)
|
||||
num_tokens += count_function(_format_function_definitions(normalized_tools))
|
||||
num_tokens += 9 # Additional tokens for function definition of tools
|
||||
# If there's a system message and tools are present, subtract four tokens
|
||||
if tools and includes_system_message:
|
||||
|
|
|
|||
|
|
@ -10512,11 +10512,13 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False)
|
|||
)
|
||||
|
||||
tokenizer_used = str(_tokenizer_used["type"])
|
||||
|
||||
total_tokens = token_counter(
|
||||
model=model_to_use,
|
||||
text=prompt,
|
||||
messages=messages,
|
||||
custom_tokenizer=_tokenizer_used, # type: ignore
|
||||
tools=tools, # type: ignore[arg-type]
|
||||
)
|
||||
return TokenCountResponse(
|
||||
total_tokens=total_tokens,
|
||||
|
|
|
|||
35
tests/proxy_unit_tests/test_count_tokens_tools.py
Normal file
35
tests/proxy_unit_tests/test_count_tokens_tools.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import pytest
|
||||
from litellm.proxy._types import TokenCountRequest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_anthropic_tools_not_ignored():
|
||||
"""Test that Anthropic-style tools are counted in fallback path"""
|
||||
from litellm.proxy.proxy_server import token_counter as proxy_token_counter
|
||||
|
||||
request_with_tools = TokenCountRequest(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=[{
|
||||
"name": "example_tool",
|
||||
"description": "A tool " + "with long description " * 100,
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"param1": {"type": "string", "description": "A parameter"}
|
||||
}
|
||||
}
|
||||
}]
|
||||
)
|
||||
|
||||
request_without_tools = TokenCountRequest(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
)
|
||||
|
||||
response_with = await proxy_token_counter(request=request_with_tools, call_endpoint=False)
|
||||
response_without = await proxy_token_counter(request=request_without_tools, call_endpoint=False)
|
||||
|
||||
assert response_with.total_tokens > response_without.total_tokens, (
|
||||
f"Tools should add tokens: with={response_with.total_tokens}, without={response_without.total_tokens}"
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue