fix(count_tokens): pass the caller's tools into the Claude count transform so advisor blocks match inference

This commit is contained in:
mateo-berri 2026-10-29 06:48:25 -07:00
parent a6f4acdb6f
commit 408372dbcb
6 changed files with 73 additions and 4 deletions

View file

@ -96,7 +96,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
pass
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.ANTHROPIC, messages, request_options, context_management, deployment
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "anthropic_api")

View file

@ -348,6 +348,7 @@ def claude_count_options(
model: str,
provider: LlmProviders,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
request_options: Mapping[str, JsonValue],
context_management: CountContextManagement | None,
deployment: Mapping[str, object] | None,
@ -371,12 +372,14 @@ def claude_count_options(
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_tools: Final = copy.deepcopy([dict(tool) for tool in tools or ()])
try:
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
messages=sent_conversation,
anthropic_messages_optional_request_params={
**copy.deepcopy(sent),
**({"tools": sent_tools} if sent_tools else {}),
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
},
litellm_params=GenericLiteLLMParams.model_validate(

View file

@ -110,7 +110,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
pass
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.AZURE_AI, messages, request_options, context_management, deployment
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")

View file

@ -187,6 +187,7 @@ class BedrockTokenCounter(BaseTokenCounter):
counted_model,
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
sent_messages,
tools if translated is None else translated.tools,
request_options,
context_management,
deployment,

View file

@ -1514,7 +1514,13 @@ class VertexAITokenCounter(BaseTokenCounter):
sent_messages: Final = messages or []
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.VERTEX_AI, sent_messages, request_options, context_management, deployment
model_to_use,
LlmProviders.VERTEX_AI,
sent_messages,
tools,
request_options,
context_management,
deployment,
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")

View file

@ -27,7 +27,7 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
@ -1000,6 +1000,65 @@ async def test_counter_sending_options_still_counts_the_leading_system_run_as_sy
}
_ADVISOR_TOOL: Final[dict[str, JsonValue]] = {
"type": ANTHROPIC_ADVISOR_TOOL_TYPE,
"name": "advisor",
"model": "claude-opus-5-5",
}
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}}
_ADVISOR_RESULT: Final[dict[str, JsonValue]] = {
"type": "advisor_tool_result",
"tool_use_id": "srvtoolu_1",
"content": {"type": "advisor_result", "text": "Check the forecast first."},
}
_ADVISED_ANSWER: Final[dict[str, JsonValue]] = {"type": "text", "text": "It is sunny."}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
@pytest.mark.parametrize(
("tools", "counted_reply"),
[
pytest.param(
[_ADVISOR_TOOL, _WEATHER_TOOL],
[_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER],
id="advisor_tool_sent",
),
pytest.param([_WEATHER_TOOL], [_ADVISED_ANSWER], id="advisor_tool_not_sent"),
],
)
async def test_counter_sending_options_keeps_advisor_blocks_only_when_the_advisor_tool_is_sent(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
tools: list[dict[str, JsonValue]],
counted_reply: list[JsonValue],
) -> None:
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[
{"role": "user", "content": "Weather in SF?"},
{"role": "assistant", "content": [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER]},
{"role": "user", "content": "And tomorrow?"},
],
tools,
"anthropic",
request_options={"thinking": _ENABLED_THINKING},
model="claude-haiku-5-5",
)
body: Final = _JSON_BODY.validate_json(count.content)
assert {name: body.get(name) for name in ("messages", "tools")} == {
"messages": [
{"role": "user", "content": "Weather in SF?"},
{"role": "assistant", "content": counted_reply},
{"role": "user", "content": "And tomorrow?"},
],
"tools": tools,
}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS