diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index bad748f8b0c..d54732883b4 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -96,7 +96,7 @@ class AnthropicTokenCounter(BaseTokenCounter): pass claude_options: Final = claude_count_options( - model_to_use, LlmProviders.ANTHROPIC, messages, request_options, context_management, deployment + model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, context_management, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "anthropic_api") diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index f816af4c187..6a9d0aa4d65 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -348,6 +348,7 @@ def claude_count_options( model: str, provider: LlmProviders, messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, request_options: Mapping[str, JsonValue], context_management: CountContextManagement | None, deployment: Mapping[str, object] | None, @@ -371,12 +372,14 @@ def claude_count_options( deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) leading_system, conversation = split_leading_system_run(messages) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) + sent_tools: Final = copy.deepcopy([dict(tool) for tool in tools or ()]) try: request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts model=model, messages=sent_conversation, anthropic_messages_optional_request_params={ **copy.deepcopy(sent), + **({"tools": sent_tools} if sent_tools else {}), "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET, }, litellm_params=GenericLiteLLMParams.model_validate( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 5bd29c9dc58..4b2fc4200ca 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -110,7 +110,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): pass claude_options: Final = claude_count_options( - model_to_use, LlmProviders.AZURE_AI, messages, request_options, context_management, deployment + model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, context_management, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api") diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 46f8828a2b9..440354dd01e 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -187,6 +187,7 @@ class BedrockTokenCounter(BaseTokenCounter): counted_model, LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, sent_messages, + tools if translated is None else translated.tools, request_options, context_management, deployment, diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index a160be3676e..7f53346ffde 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1514,7 +1514,13 @@ class VertexAITokenCounter(BaseTokenCounter): sent_messages: Final = messages or [] claude_options: Final = claude_count_options( - model_to_use, LlmProviders.VERTEX_AI, sent_messages, request_options, context_management, deployment + model_to_use, + LlmProviders.VERTEX_AI, + sent_messages, + tools, + request_options, + context_management, + deployment, ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models") diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 8b4e514a82f..9e818fae5c4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -27,7 +27,7 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse @@ -1000,6 +1000,65 @@ async def test_counter_sending_options_still_counts_the_leading_system_run_as_sy } +_ADVISOR_TOOL: Final[dict[str, JsonValue]] = { + "type": ANTHROPIC_ADVISOR_TOOL_TYPE, + "name": "advisor", + "model": "claude-opus-5-5", +} +_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}} +_ADVISOR_RESULT: Final[dict[str, JsonValue]] = { + "type": "advisor_tool_result", + "tool_use_id": "srvtoolu_1", + "content": {"type": "advisor_result", "text": "Check the forecast first."}, +} +_ADVISED_ANSWER: Final[dict[str, JsonValue]] = {"type": "text", "text": "It is sunny."} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + ("tools", "counted_reply"), + [ + pytest.param( + [_ADVISOR_TOOL, _WEATHER_TOOL], + [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER], + id="advisor_tool_sent", + ), + pytest.param([_WEATHER_TOOL], [_ADVISED_ANSWER], id="advisor_tool_not_sent"), + ], +) +async def test_counter_sending_options_keeps_advisor_blocks_only_when_the_advisor_tool_is_sent( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + tools: list[dict[str, JsonValue]], + counted_reply: list[JsonValue], +) -> None: + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [ + {"role": "user", "content": "Weather in SF?"}, + {"role": "assistant", "content": [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER]}, + {"role": "user", "content": "And tomorrow?"}, + ], + tools, + "anthropic", + request_options={"thinking": _ENABLED_THINKING}, + model="claude-haiku-5-5", + ) + + body: Final = _JSON_BODY.validate_json(count.content) + assert {name: body.get(name) for name in ("messages", "tools")} == { + "messages": [ + {"role": "user", "content": "Weather in SF?"}, + {"role": "assistant", "content": counted_reply}, + {"role": "user", "content": "And tomorrow?"}, + ], + "tools": tools, + } + + @pytest.mark.asyncio @pytest.mark.usefixtures("httpx_transport_clients") @_COUNTERS