fix(count_tokens): count the messages the Bedrock transform sends when it inlines the output schema

This commit is contained in:
mateo-berri 2026-10-29 06:17:53 -07:00
parent 875a0e6404
commit a6f4acdb6f
7 changed files with 114 additions and 12 deletions

View file

@ -96,7 +96,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
pass
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.ANTHROPIC, request_options, context_management, deployment
model_to_use, LlmProviders.ANTHROPIC, messages, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "anthropic_api")
@ -133,7 +133,9 @@ class AnthropicTokenCounter(BaseTokenCounter):
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,
messages=messages,
messages=messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
auth_header=auth_header,
api_base=api_base,
tools=tools,

View file

@ -129,7 +129,6 @@ class _AnthropicChatRequest(_CountedOptions):
_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest)
_COUNTED_OPTIONS: Final = TypeAdapter(_CountedOptions)
_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object])
_ChatParams: TypeAlias = dict[str, object]
_ChatMessages: TypeAlias = list[AllMessageValues]
@ -337,15 +336,18 @@ def anthropic_fields_for_chat_tool_use(
class ClaudeCountOptions:
options: Mapping[str, JsonValue]
betas: tuple[str, ...]
messages: tuple[Mapping[str, JsonValue], ...] | None
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=())
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None)
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
def claude_count_options(
model: str,
provider: LlmProviders,
messages: Sequence[Mapping[str, object]],
request_options: Mapping[str, JsonValue],
context_management: CountContextManagement | None,
deployment: Mapping[str, object] | None,
@ -367,10 +369,12 @@ def claude_count_options(
return NO_CLAUDE_COUNT_OPTIONS
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
try:
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
messages=[],
messages=sent_conversation,
anthropic_messages_optional_request_params={
**copy.deepcopy(sent),
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
@ -385,7 +389,8 @@ def claude_count_options(
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
options: Final = _count_options(_COUNTED_OPTIONS.validate_python(request))
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
options: Final = _count_options(transformed)
betas: Final = sorted(context_management_betas(options.get("context_management")))
return ClaudeCountOptions(
options=options,
@ -394,6 +399,7 @@ def claude_count_options(
if messages_config.should_filter_anthropic_beta_headers()
else betas
),
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
)

View file

@ -110,7 +110,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
pass
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.AZURE_AI, request_options, context_management, deployment
model_to_use, LlmProviders.AZURE_AI, messages, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
@ -136,7 +136,9 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
try:
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,
messages=messages,
messages=messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
api_key=api_key,
api_base=api_base,
litellm_params=litellm_params,

View file

@ -181,10 +181,12 @@ class BedrockTokenCounter(BaseTokenCounter):
)
if isinstance(translated, ChatCountTranslationError):
return translated.response(request_model, model_to_use, primary_tokenizer)
sent_messages: Final = messages if translated is None else translated.messages
claude_options: Final = (
claude_count_options(
counted_model,
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
sent_messages,
request_options,
context_management,
deployment,
@ -194,11 +196,12 @@ class BedrockTokenCounter(BaseTokenCounter):
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, primary_tokenizer)
counted_messages: Final = sent_messages if claude_options.messages is None else claude_options.messages
runtime_request: Final = {
**(
_request_data(model_to_use, messages, tools, system)
_request_data(model_to_use, counted_messages, tools, system)
if translated is None
else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
else _request_data(model_to_use, counted_messages, translated.tools, translated.system)
),
**_invoke_fields(claude_options),
}

View file

@ -1512,8 +1512,9 @@ class VertexAITokenCounter(BaseTokenCounter):
claude_count_options,
)
sent_messages: Final = messages or []
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.VERTEX_AI, request_options, context_management, deployment
model_to_use, LlmProviders.VERTEX_AI, sent_messages, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
@ -1541,7 +1542,9 @@ class VertexAITokenCounter(BaseTokenCounter):
try:
result = await partner_models_handler.count_tokens(
model=model_to_use,
messages=messages or [],
messages=sent_messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
litellm_params=partner_litellm_params,
vertex_project=vertex_project,
vertex_location=vertex_location,

View file

@ -976,6 +976,30 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_sending_options_still_counts_the_leading_system_run_as_system(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "system", "content": "Be terse."}, {"role": "user", "content": "Weather in SF?"}],
None,
"anthropic",
request_options={"thinking": _ENABLED_THINKING},
model="claude-haiku-5-5",
)
body: Final = _JSON_BODY.validate_json(count.content)
assert "thinking" in body
assert {name: body.get(name) for name in ("system", "messages")} == {
"system": [{"type": "text", "text": "Be terse."}],
"messages": [{"role": "user", "content": "Weather in SF?"}],
}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS

View file

@ -1,4 +1,5 @@
import base64
import json
from collections.abc import Mapping
from typing import Final
@ -418,6 +419,67 @@ async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_infe
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
_CITY_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
def _texts(message: JsonValue) -> tuple[str, ...]:
content: Final = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
return ()
return tuple(text for block in content if isinstance(block, dict) and isinstance(text := block.get("text"), str))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("native_output", "request_options", "schema_inlined"),
(
(False, _CITY_FORMAT, True),
(True, _CITY_FORMAT, False),
(False, _THINKING_AND_TOOL_CHOICE, False),
),
ids=("inlined_schema", "native_schema", "no_format"),
)
async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends(
native_output: bool, request_options: dict[str, JsonValue], schema_inlined: bool
) -> None:
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-count-probe-v1:0"
model_info: Final = {"supports_bedrock_native_structured_output": native_output}
credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages(
model=deployment_model,
messages=MESSAGES,
max_tokens=4096,
model_info=model_info,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**request_options,
**credentials,
)
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-count-probe-v1:0",
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
request_model="claude",
request_options=request_options,
)
assert result is not None
assert (result.error, result.total_tokens) == (False, 412)
counted: Final = _invoke_body(bedrock.requests[0])["messages"]
assert counted == _JSON_BODY.validate_json(inference.requests[0].content)["messages"]
assert isinstance(counted, list)
assert (json.dumps(_CITY_SCHEMA) in _texts(counted[-1])) is schema_inlined
assert (counted == MESSAGES) is not schema_inlined
@pytest.mark.asyncio
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))