From a6f4acdb6f3651cdc2792aef8b0989db5f15a906 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:17:53 -0700 Subject: [PATCH] fix(count_tokens): count the messages the Bedrock transform sends when it inlines the output schema --- .../anthropic/count_tokens/token_counter.py | 6 +- .../anthropic/count_tokens/transformation.py | 14 +++-- .../anthropic/count_tokens/token_counter.py | 6 +- .../count_tokens/bedrock_token_counter.py | 7 ++- litellm/llms/vertex_ai/common_utils.py | 7 ++- ...t_anthropic_count_tokens_transformation.py | 24 +++++++ .../test_bedrock_token_counter.py | 62 +++++++++++++++++++ 7 files changed, 114 insertions(+), 12 deletions(-) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 0498b00e1e6..bad748f8b0c 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -96,7 +96,7 @@ class AnthropicTokenCounter(BaseTokenCounter): pass claude_options: Final = claude_count_options( - model_to_use, LlmProviders.ANTHROPIC, request_options, context_management, deployment + model_to_use, LlmProviders.ANTHROPIC, messages, request_options, context_management, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "anthropic_api") @@ -133,7 +133,9 @@ class AnthropicTokenCounter(BaseTokenCounter): result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, - messages=messages, + messages=messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], auth_header=auth_header, api_base=api_base, tools=tools, diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index e7e6ba5c8ce..f816af4c187 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -129,7 +129,6 @@ class _AnthropicChatRequest(_CountedOptions): _ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest) -_COUNTED_OPTIONS: Final = TypeAdapter(_CountedOptions) _INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object]) _ChatParams: TypeAlias = dict[str, object] _ChatMessages: TypeAlias = list[AllMessageValues] @@ -337,15 +336,18 @@ def anthropic_fields_for_chat_tool_use( class ClaudeCountOptions: options: Mapping[str, JsonValue] betas: tuple[str, ...] + messages: tuple[Mapping[str, JsonValue], ...] | None -NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=()) +NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None) +_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) _MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1 def claude_count_options( model: str, provider: LlmProviders, + messages: Sequence[Mapping[str, object]], request_options: Mapping[str, JsonValue], context_management: CountContextManagement | None, deployment: Mapping[str, object] | None, @@ -367,10 +369,12 @@ def claude_count_options( return NO_CLAUDE_COUNT_OPTIONS resolved_deployment: Final = deployment or {} deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + leading_system, conversation = split_leading_system_run(messages) + sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) try: request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts model=model, - messages=[], + messages=sent_conversation, anthropic_messages_optional_request_params={ **copy.deepcopy(sent), "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET, @@ -385,7 +389,8 @@ def claude_count_options( ) except (AnthropicError, BadRequestError) as e: return ChatCountTranslationError(message=str(e.message)) - options: Final = _count_options(_COUNTED_OPTIONS.validate_python(request)) + transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) + options: Final = _count_options(transformed) betas: Final = sorted(context_management_betas(options.get("context_management"))) return ClaudeCountOptions( options=options, @@ -394,6 +399,7 @@ def claude_count_options( if messages_config.should_filter_anthropic_beta_headers() else betas ), + messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]), ) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 4311b56f9b2..5bd29c9dc58 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -110,7 +110,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): pass claude_options: Final = claude_count_options( - model_to_use, LlmProviders.AZURE_AI, request_options, context_management, deployment + model_to_use, LlmProviders.AZURE_AI, messages, request_options, context_management, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api") @@ -136,7 +136,9 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): try: result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, - messages=messages, + messages=messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], api_key=api_key, api_base=api_base, litellm_params=litellm_params, diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 18b6caf390f..46f8828a2b9 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -181,10 +181,12 @@ class BedrockTokenCounter(BaseTokenCounter): ) if isinstance(translated, ChatCountTranslationError): return translated.response(request_model, model_to_use, primary_tokenizer) + sent_messages: Final = messages if translated is None else translated.messages claude_options: Final = ( claude_count_options( counted_model, LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, + sent_messages, request_options, context_management, deployment, @@ -194,11 +196,12 @@ class BedrockTokenCounter(BaseTokenCounter): ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, primary_tokenizer) + counted_messages: Final = sent_messages if claude_options.messages is None else claude_options.messages runtime_request: Final = { **( - _request_data(model_to_use, messages, tools, system) + _request_data(model_to_use, counted_messages, tools, system) if translated is None - else _request_data(model_to_use, translated.messages, translated.tools, translated.system) + else _request_data(model_to_use, counted_messages, translated.tools, translated.system) ), **_invoke_fields(claude_options), } diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 44bb60573c4..a160be3676e 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1512,8 +1512,9 @@ class VertexAITokenCounter(BaseTokenCounter): claude_count_options, ) + sent_messages: Final = messages or [] claude_options: Final = claude_count_options( - model_to_use, LlmProviders.VERTEX_AI, request_options, context_management, deployment + model_to_use, LlmProviders.VERTEX_AI, sent_messages, request_options, context_management, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models") @@ -1541,7 +1542,9 @@ class VertexAITokenCounter(BaseTokenCounter): try: result = await partner_models_handler.count_tokens( model=model_to_use, - messages=messages or [], + messages=sent_messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], litellm_params=partner_litellm_params, vertex_project=vertex_project, vertex_location=vertex_location, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 51fff399f5e..8b4e514a82f 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -976,6 +976,30 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_sending_options_still_counts_the_leading_system_run_as_system( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "system", "content": "Be terse."}, {"role": "user", "content": "Weather in SF?"}], + None, + "anthropic", + request_options={"thinking": _ENABLED_THINKING}, + model="claude-haiku-5-5", + ) + + body: Final = _JSON_BODY.validate_json(count.content) + assert "thinking" in body + assert {name: body.get(name) for name in ("system", "messages")} == { + "system": [{"type": "text", "text": "Be terse."}], + "messages": [{"role": "user", "content": "Weather in SF?"}], + } + + @pytest.mark.asyncio @pytest.mark.usefixtures("httpx_transport_clients") @_COUNTERS diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 8a6a1242b95..d3534dd39b9 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -1,4 +1,5 @@ import base64 +import json from collections.abc import Mapping from typing import Final @@ -418,6 +419,67 @@ async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_infe assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"] +_CITY_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}} + + +def _texts(message: JsonValue) -> tuple[str, ...]: + content: Final = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + return () + return tuple(text for block in content if isinstance(block, dict) and isinstance(text := block.get("text"), str)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("native_output", "request_options", "schema_inlined"), + ( + (False, _CITY_FORMAT, True), + (True, _CITY_FORMAT, False), + (False, _THINKING_AND_TOOL_CHOICE, False), + ), + ids=("inlined_schema", "native_schema", "no_format"), +) +async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends( + native_output: bool, request_options: dict[str, JsonValue], schema_inlined: bool +) -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-count-probe-v1:0" + model_info: Final = {"supports_bedrock_native_structured_output": native_output} + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + max_tokens=4096, + model_info=model_info, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + **request_options, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-count-probe-v1:0", + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info}, + request_model="claude", + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 412) + counted: Final = _invoke_body(bedrock.requests[0])["messages"] + assert counted == _JSON_BODY.validate_json(inference.requests[0].content)["messages"] + assert isinstance(counted, list) + assert (json.dumps(_CITY_SCHEMA) in _texts(counted[-1])) is schema_inlined + assert (counted == MESSAGES) is not schema_inlined + + @pytest.mark.asyncio async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None: bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))