mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): pass the caller's tools into the Claude count transform so advisor blocks match inference
This commit is contained in:
parent
a6f4acdb6f
commit
408372dbcb
6 changed files with 73 additions and 4 deletions
|
|
@ -96,7 +96,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
claude_options: Final = claude_count_options(
|
claude_options: Final = claude_count_options(
|
||||||
model_to_use, LlmProviders.ANTHROPIC, messages, request_options, context_management, deployment
|
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, context_management, deployment
|
||||||
)
|
)
|
||||||
if isinstance(claude_options, ChatCountTranslationError):
|
if isinstance(claude_options, ChatCountTranslationError):
|
||||||
return claude_options.response(request_model, model_to_use, "anthropic_api")
|
return claude_options.response(request_model, model_to_use, "anthropic_api")
|
||||||
|
|
|
||||||
|
|
@ -348,6 +348,7 @@ def claude_count_options(
|
||||||
model: str,
|
model: str,
|
||||||
provider: LlmProviders,
|
provider: LlmProviders,
|
||||||
messages: Sequence[Mapping[str, object]],
|
messages: Sequence[Mapping[str, object]],
|
||||||
|
tools: Sequence[Mapping[str, object]] | None,
|
||||||
request_options: Mapping[str, JsonValue],
|
request_options: Mapping[str, JsonValue],
|
||||||
context_management: CountContextManagement | None,
|
context_management: CountContextManagement | None,
|
||||||
deployment: Mapping[str, object] | None,
|
deployment: Mapping[str, object] | None,
|
||||||
|
|
@ -371,12 +372,14 @@ def claude_count_options(
|
||||||
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||||
leading_system, conversation = split_leading_system_run(messages)
|
leading_system, conversation = split_leading_system_run(messages)
|
||||||
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
||||||
|
sent_tools: Final = copy.deepcopy([dict(tool) for tool in tools or ()])
|
||||||
try:
|
try:
|
||||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||||
model=model,
|
model=model,
|
||||||
messages=sent_conversation,
|
messages=sent_conversation,
|
||||||
anthropic_messages_optional_request_params={
|
anthropic_messages_optional_request_params={
|
||||||
**copy.deepcopy(sent),
|
**copy.deepcopy(sent),
|
||||||
|
**({"tools": sent_tools} if sent_tools else {}),
|
||||||
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
|
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
|
||||||
},
|
},
|
||||||
litellm_params=GenericLiteLLMParams.model_validate(
|
litellm_params=GenericLiteLLMParams.model_validate(
|
||||||
|
|
|
||||||
|
|
@ -110,7 +110,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
claude_options: Final = claude_count_options(
|
claude_options: Final = claude_count_options(
|
||||||
model_to_use, LlmProviders.AZURE_AI, messages, request_options, context_management, deployment
|
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, context_management, deployment
|
||||||
)
|
)
|
||||||
if isinstance(claude_options, ChatCountTranslationError):
|
if isinstance(claude_options, ChatCountTranslationError):
|
||||||
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
||||||
|
|
|
||||||
|
|
@ -187,6 +187,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
||||||
counted_model,
|
counted_model,
|
||||||
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
|
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
|
||||||
sent_messages,
|
sent_messages,
|
||||||
|
tools if translated is None else translated.tools,
|
||||||
request_options,
|
request_options,
|
||||||
context_management,
|
context_management,
|
||||||
deployment,
|
deployment,
|
||||||
|
|
|
||||||
|
|
@ -1514,7 +1514,13 @@ class VertexAITokenCounter(BaseTokenCounter):
|
||||||
|
|
||||||
sent_messages: Final = messages or []
|
sent_messages: Final = messages or []
|
||||||
claude_options: Final = claude_count_options(
|
claude_options: Final = claude_count_options(
|
||||||
model_to_use, LlmProviders.VERTEX_AI, sent_messages, request_options, context_management, deployment
|
model_to_use,
|
||||||
|
LlmProviders.VERTEX_AI,
|
||||||
|
sent_messages,
|
||||||
|
tools,
|
||||||
|
request_options,
|
||||||
|
context_management,
|
||||||
|
deployment,
|
||||||
)
|
)
|
||||||
if isinstance(claude_options, ChatCountTranslationError):
|
if isinstance(claude_options, ChatCountTranslationError):
|
||||||
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
|
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
|
||||||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||||
AzureAIAnthropicCountTokensConfig,
|
AzureAIAnthropicCountTokensConfig,
|
||||||
)
|
)
|
||||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES
|
||||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1000,6 +1000,65 @@ async def test_counter_sending_options_still_counts_the_leading_system_run_as_sy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_ADVISOR_TOOL: Final[dict[str, JsonValue]] = {
|
||||||
|
"type": ANTHROPIC_ADVISOR_TOOL_TYPE,
|
||||||
|
"name": "advisor",
|
||||||
|
"model": "claude-opus-5-5",
|
||||||
|
}
|
||||||
|
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}}
|
||||||
|
_ADVISOR_RESULT: Final[dict[str, JsonValue]] = {
|
||||||
|
"type": "advisor_tool_result",
|
||||||
|
"tool_use_id": "srvtoolu_1",
|
||||||
|
"content": {"type": "advisor_result", "text": "Check the forecast first."},
|
||||||
|
}
|
||||||
|
_ADVISED_ANSWER: Final[dict[str, JsonValue]] = {"type": "text", "text": "It is sunny."}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||||
|
@_COUNTERS
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("tools", "counted_reply"),
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
[_ADVISOR_TOOL, _WEATHER_TOOL],
|
||||||
|
[_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER],
|
||||||
|
id="advisor_tool_sent",
|
||||||
|
),
|
||||||
|
pytest.param([_WEATHER_TOOL], [_ADVISED_ANSWER], id="advisor_tool_not_sent"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_counter_sending_options_keeps_advisor_blocks_only_when_the_advisor_tool_is_sent(
|
||||||
|
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||||
|
count_endpoint: str,
|
||||||
|
tools: list[dict[str, JsonValue]],
|
||||||
|
counted_reply: list[JsonValue],
|
||||||
|
) -> None:
|
||||||
|
_, (count,) = await _counted_requests(
|
||||||
|
counter_type,
|
||||||
|
count_endpoint,
|
||||||
|
[
|
||||||
|
{"role": "user", "content": "Weather in SF?"},
|
||||||
|
{"role": "assistant", "content": [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER]},
|
||||||
|
{"role": "user", "content": "And tomorrow?"},
|
||||||
|
],
|
||||||
|
tools,
|
||||||
|
"anthropic",
|
||||||
|
request_options={"thinking": _ENABLED_THINKING},
|
||||||
|
model="claude-haiku-5-5",
|
||||||
|
)
|
||||||
|
|
||||||
|
body: Final = _JSON_BODY.validate_json(count.content)
|
||||||
|
assert {name: body.get(name) for name in ("messages", "tools")} == {
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "Weather in SF?"},
|
||||||
|
{"role": "assistant", "content": counted_reply},
|
||||||
|
{"role": "user", "content": "And tomorrow?"},
|
||||||
|
],
|
||||||
|
"tools": tools,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||||
@_COUNTERS
|
@_COUNTERS
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue