fix(count_tokens): count with the deployment's model, add the effort beta on Bedrock runtime counts, and match what Converse sends

The count now transforms with the deployment's own inference model, falling back to the counted model, so a stand-in
Mantle count sees the native output_config.format the deployment sends. Bedrock runtime CountTokens refuses
output_config.effort without effort-2025-11-24, so the runtime count adds it while Mantle keeps inference's betas.
A Converse deployment's count leaves out tool_choice.disable_parallel_tool_use, which the Converse bridge never sends.
Only the transform's own errors on the caller's input become the 400 that counts again without the options, and the
request's drop_params is normalized the way inference reads it
This commit is contained in:
mateo-berri 2026-11-06 19:44:41 -08:00
parent 14b572d44b
commit 502ce4a887
5 changed files with 256 additions and 57 deletions

View file

@ -10,6 +10,7 @@ from dataclasses import dataclass
from itertools import chain from itertools import chain
from types import MappingProxyType from types import MappingProxyType
from typing import ( from typing import (
TYPE_CHECKING,
Final, Final,
Protocol, Protocol,
TypeAlias, TypeAlias,
@ -32,6 +33,9 @@ from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropi
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
if TYPE_CHECKING:
from litellm.types.router import GenericLiteLLMParams
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject) _IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
@ -353,21 +357,55 @@ def is_request_rejection(status_code: int) -> bool:
return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST
def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None:
deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {})
deployment_model: Final = deployment_params.get("model")
if not isinstance(deployment_model, str):
return None
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
provider_hint: Final = deployment_params.get("custom_llm_provider")
api_base: Final = deployment_params.get("api_base")
inference_model, _, _, _ = get_llm_provider(
model=deployment_model,
custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)
return inference_model
def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None:
from litellm.utils import ProviderConfigManager
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
return None if messages_config is None else (model, messages_config)
def _transform_litellm_params(deployment: Mapping[str, object] | None) -> "GenericLiteLLMParams":
from litellm.types.router import GenericLiteLLMParams
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
return GenericLiteLLMParams.model_validate(
{
"drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"),
}
)
def _claude_count_options_for( def _claude_count_options_for(
messages_config: BaseAnthropicMessagesConfig, messages_config: BaseAnthropicMessagesConfig,
model: str, model: str,
litellm_params: "GenericLiteLLMParams",
provider: LlmProviders, provider: LlmProviders,
messages: Sequence[Mapping[str, object]], messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None, tools: Sequence[Mapping[str, object]] | None,
sent: Mapping[str, JsonValue], sent: Mapping[str, JsonValue],
anthropic_beta: str | None, anthropic_beta: str | None,
deployment: Mapping[str, object] | None,
) -> ClaudeCountOptions: ) -> ClaudeCountOptions:
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.types.router import GenericLiteLLMParams
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
leading_system, conversation = split_leading_system_run(messages) leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_params: Final = { sent_params: Final = {
@ -386,12 +424,7 @@ def _claude_count_options_for(
model=model, model=model,
messages=sent_conversation, messages=sent_conversation,
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET}, anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
litellm_params=GenericLiteLLMParams.model_validate( litellm_params=litellm_params,
{
"drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"),
}
),
headers=sent_headers, headers=sent_headers,
) )
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
@ -415,20 +448,24 @@ def claude_count_options(
sent: Final = {name: value for name, value in request_options.items() if value is not None} sent: Final = {name: value for name, value in request_options.items() if value is not None}
if not sent: if not sent:
return NO_CLAUDE_COUNT_OPTIONS return NO_CLAUDE_COUNT_OPTIONS
inference_model: Final = deployment_inference_model(deployment)
transform: Final = (
None if inference_model is None else _messages_transform(inference_model, provider)
) or _messages_transform(model, provider)
if transform is None:
return NO_CLAUDE_COUNT_OPTIONS
transform_model, messages_config = transform
litellm_params: Final = _transform_litellm_params(deployment)
from litellm.exceptions import BadRequestError from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.utils import ProviderConfigManager
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
if messages_config is None:
return NO_CLAUDE_COUNT_OPTIONS
try: try:
return _claude_count_options_for( return _claude_count_options_for(
messages_config, model, provider, messages, tools, sent, anthropic_beta, deployment messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
) )
except (AnthropicError, BadRequestError) as e: except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message)) return ChatCountTranslationError(message=str(e.message))
except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e: except (AttributeError, KeyError, TypeError, ValueError) as e:
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")

View file

@ -19,6 +19,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
ClaudeCountOptions, ClaudeCountOptions,
anthropic_count_fields, anthropic_count_fields,
claude_count_options, claude_count_options,
deployment_inference_model,
is_request_rejection, is_request_rejection,
) )
from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.base_llm.base_utils import BaseTokenCounter
@ -27,6 +28,7 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER
from litellm.types.utils import ( from litellm.types.utils import (
CountContextManagement, CountContextManagement,
LiteLLMPydanticObjectBase, LiteLLMPydanticObjectBase,
@ -85,10 +87,41 @@ def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options
return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code) return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code)
def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]: def _invoke_fields(options: Mapping[str, JsonValue], betas: Sequence[str]) -> Mapping[str, object]:
return {**options, **({"anthropic_beta": list(betas)} if betas else {})}
def _sets_effort(options: Mapping[str, JsonValue]) -> bool:
output_config: Final = options.get("output_config")
return isinstance(output_config, dict) and "effort" in output_config
def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]:
if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas:
return claude_options.betas
return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER)
def _runs_on_converse(inference_model: str) -> bool:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(inference_model) is None
def _options_converse_sends(
request_options: Mapping[str, JsonValue], inference_model: str | None
) -> Mapping[str, JsonValue]:
tool_choice: Final = request_options.get("tool_choice")
if (
not isinstance(tool_choice, dict)
or "disable_parallel_tool_use" not in tool_choice
or inference_model is None
or not _runs_on_converse(inference_model)
):
return request_options
return { return {
**claude_options.options, **request_options,
**({"anthropic_beta": list(claude_options.betas)} if claude_options.betas else {}), "tool_choice": {key: value for key, value in tool_choice.items() if key != "disable_parallel_tool_use"},
} }
@ -193,7 +226,7 @@ class BedrockTokenCounter(BaseTokenCounter):
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
sent_messages, sent_messages,
sent_tools, sent_tools,
request_options, _options_converse_sends(request_options, deployment_inference_model(deployment)),
anthropic_beta, anthropic_beta,
deployment, deployment,
) )
@ -202,17 +235,22 @@ class BedrockTokenCounter(BaseTokenCounter):
) )
if isinstance(claude_options, ChatCountTranslationError): if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, primary_tokenizer) return claude_options.response(request_model, model_to_use, primary_tokenizer)
invoke_fields: Final = _invoke_fields(claude_options) invoke_fields: Final = _invoke_fields(claude_options.options, claude_options.betas)
counted_request: Final = _request_data(
model_to_use,
sent_messages if claude_options.messages is None else claude_options.messages,
sent_tools if claude_options.messages is None else claude_options.tools,
system if translated is None else translated.system,
)
runtime_request: Final = { runtime_request: Final = {
**_request_data( **counted_request,
model_to_use, **_invoke_fields(claude_options.options, _runtime_count_betas(claude_options)),
sent_messages if claude_options.messages is None else claude_options.messages, }
sent_tools if claude_options.messages is None else claude_options.tools, mantle_request: Final = {
system if translated is None else translated.system, **counted_request,
), **invoke_fields,
**invoke_fields, **({} if translated is None else translated.options),
} }
mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
primary: Final = ( primary: Final = (
await self._count_with( await self._count_with(

View file

@ -14943,9 +14943,9 @@ async def _count_retrying_without_options(
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
return counted return counted
verbose_proxy_logger.warning( verbose_proxy_logger.warning(
"Token count for %s answered %s with the request's %s, so it counts again without them", "Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",
model, model,
counted.status_code, "could not be built" if counted.rejected_before_provider_call else f"answered {counted.status_code}",
", ".join(forwarded_options), ", ".join(forwarded_options),
) )
return await count_with(model, region, request_options=MappingProxyType({})) return await count_with(model, region, request_options=MappingProxyType({}))

View file

@ -353,6 +353,14 @@ _THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}} _WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
_CITY_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS} _CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
@ -362,9 +370,10 @@ async def _messages_inference_and_count(
request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS, request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS,
tool: Mapping[str, JsonValue] = _WEATHER_TOOL, tool: Mapping[str, JsonValue] = _WEATHER_TOOL,
runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}), runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}),
inference_reply: Mapping[str, object] = _CLAUDE_REPLY,
) -> tuple[_Bedrock, _Bedrock]: ) -> tuple[_Bedrock, _Bedrock]:
credentials: Final = DEPLOYMENT["litellm_params"] credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) inference: Final = _Bedrock(runtime=(200, inference_reply), mantle=(200, inference_reply))
await litellm.anthropic_messages( await litellm.anthropic_messages(
model=deployment_model, model=deployment_model,
messages=MESSAGES, messages=MESSAGES,
@ -389,11 +398,17 @@ async def _messages_inference_and_count(
return inference, bedrock return inference, bedrock
def _betas_besides_effort(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]:
betas: Final = body.get("anthropic_beta") or []
assert isinstance(betas, list)
return tuple(beta for beta in betas if beta != ANTHROPIC_EFFORT_BETA_HEADER)
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize( @pytest.mark.parametrize(
("deployment_model", "model_to_use"), ("deployment_model", "model_to_use"),
( (
("bedrock/invoke/global.anthropic.claude-haiku-5-5", "global.anthropic.claude-haiku-5-5"), ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-sonnet-4-6"),
("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"), ("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"),
), ),
) )
@ -405,8 +420,9 @@ async def test_runtime_count_sends_the_options_and_context_edits_invoke_inferenc
assert bedrock.posted_hosts() == (RUNTIME_HOST,) assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _invoke_body(bedrock.requests[0]) counted: Final = _invoke_body(bedrock.requests[0])
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta") fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
assert _betas_besides_effort(counted) == _betas_besides_effort(sent)
thinking: Final = counted["thinking"] thinking: Final = counted["thinking"]
assert isinstance(thinking, dict) assert isinstance(thinking, dict)
budget_tokens: Final = thinking.get("budget_tokens", 0) budget_tokens: Final = thinking.get("budget_tokens", 0)
@ -433,7 +449,7 @@ _DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive
def _sent_betas(request: httpx.Request) -> tuple[str, ...]: def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
return tuple(sorted(request.headers["anthropic-beta"].split(","))) return tuple(sorted(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta))
@pytest.mark.asyncio @pytest.mark.asyncio
@ -462,20 +478,109 @@ async def test_runtime_count_sends_the_betas_invoke_inference_sends(
assert isinstance(counted_betas, list) and expected_beta in counted_betas assert isinstance(counted_betas, list) and expected_beta in counted_betas
_EFFORT: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}}
_CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = {
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]}
}
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends() -> None: @pytest.mark.parametrize("request_options", (_DISPLAY_THINKING, _EFFORT), ids=("thinking_display", "effort"))
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends(
request_options: dict[str, JsonValue],
) -> None:
inference, bedrock = await _messages_inference_and_count( inference, bedrock = await _messages_inference_and_count(
"bedrock/invoke/us.anthropic.claude-opus-5-5", "bedrock/invoke/us.anthropic.claude-opus-5-5",
"us.anthropic.claude-opus-5-5", "us.anthropic.claude-opus-5-5",
_DISPLAY_THINKING, request_options,
runtime=(400, UNSUPPORTED), runtime=(400, UNSUPPORTED),
) )
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta") sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", [])
assert isinstance(sent_betas, list) assert isinstance(sent_betas, list)
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas))
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in sent_betas
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "request_options"),
(
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _EFFORT),
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLEAR_THINKING_EDITS),
("bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", _EFFORT),
),
ids=("effort", "clear_thinking_sets_effort", "inference_sends_the_effort_beta"),
)
async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_betas_invoke_inference_sends(
deployment_model: str, request_options: dict[str, JsonValue]
) -> None:
inference, bedrock = await _messages_inference_and_count(
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _invoke_body(bedrock.requests[0])
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
counted_config: Final = counted.get("output_config")
assert isinstance(counted_config, dict) and counted_config.get("effort") == "low"
counted_betas: Final = counted.get("anthropic_beta")
sent_betas: Final = sent.get("anthropic_beta", [])
assert isinstance(counted_betas, list) and isinstance(sent_betas, list)
assert counted_betas.count(ANTHROPIC_EFFORT_BETA_HEADER) == 1
assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER}
_CONVERSE_REPLY: Final = {
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4},
"metrics": {"latencyMs": 1},
}
_NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = {
"tool_choice": {"type": "auto", "disable_parallel_tool_use": True}
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "inference_reply"),
(
("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY),
("bedrock/converse/us.anthropic.claude-opus-4-6-v1", _CONVERSE_REPLY),
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY),
),
ids=("converse", "converse_parallel_tool_use_config_model", "invoke"),
)
async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it(
deployment_model: str, inference_reply: Mapping[str, object]
) -> None:
inference, bedrock = await _messages_inference_and_count(
deployment_model,
deployment_model.split("/")[-1],
_NO_PARALLEL_TOOL_USE,
inference_reply=inference_reply,
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted_tool_choice: Final = _invoke_body(bedrock.requests[0]).get("tool_choice")
assert isinstance(counted_tool_choice, dict) and counted_tool_choice.get("type") == "auto"
assert ("disable_parallel_tool_use" in counted_tool_choice) == (
b"disable_parallel_tool_use" in inference.requests[0].content
)
@pytest.mark.asyncio
async def test_count_on_a_stand_in_model_counts_the_messages_and_output_config_the_deployment_model_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/us.anthropic.claude-haiku-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _CITY_FORMAT
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("messages", "output_config")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
@pytest.mark.asyncio @pytest.mark.asyncio
@ -573,14 +678,6 @@ async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
_CITY_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
def _texts(message: JsonValue) -> tuple[str, ...]: def _texts(message: JsonValue) -> tuple[str, ...]:
content: Final = message.get("content") if isinstance(message, dict) else None content: Final = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list): if not isinstance(content, list):
@ -634,17 +731,34 @@ async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends(
assert (counted == MESSAGES) is not schema_inlined assert (counted == MESSAGES) is not schema_inlined
_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}}
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None: @pytest.mark.parametrize(
("messages", "tools", "request_options"),
(
(MESSAGES, None, {"output_config": {"effort": "fastest"}}),
(MESSAGES, [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], _THINKING),
(MESSAGES, ["get_weather"], _THINKING),
([{"role": "user", "content": 5}], None, _THINKING),
([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING),
),
ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"),
)
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock(
messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue]
) -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens( result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0", model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
messages=MESSAGES, messages=messages,
contents=None, contents=None,
deployment=DEPLOYMENT, deployment=DEPLOYMENT,
request_model="claude-sonnet-4-5", request_model="claude-sonnet-4-5",
request_options={"output_config": {"effort": "fastest"}}, tools=tools,
request_options=request_options,
) )
assert result is not None assert result is not None

View file

@ -1915,15 +1915,21 @@ def _mantle_unreachable(request: httpx.Request) -> httpx.Response:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_tool_again_without_the_options( @pytest.mark.parametrize(
monkeypatch: pytest.MonkeyPatch, "malformed",
(
{"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}]},
{"messages": [{"role": "user", "content": 5}]},
{"messages": [{"role": "user", "content": [{"text": "Weather in SF?"}]}]},
),
ids=("tool_type_not_a_string", "content_not_iterable", "block_without_type"),
)
async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_request_again_without_the_options(
monkeypatch: pytest.MonkeyPatch, malformed: dict[str, JsonValue]
) -> None: ) -> None:
response, requests = await _count_on_a_bedrock_claude_deployment( response, requests = await _count_on_a_bedrock_claude_deployment(
monkeypatch, monkeypatch,
{ {**malformed, "thinking": {"type": "enabled", "budget_tokens": 2000}},
"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}],
"thinking": {"type": "enabled", "budget_tokens": 2000},
},
_runtime_counts_only_without_options, _runtime_counts_only_without_options,
_mantle_unreachable, _mantle_unreachable,
) )
@ -2070,7 +2076,11 @@ async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize( @pytest.mark.parametrize(
("request_drop_params", "user_agent"), ("request_drop_params", "user_agent"),
(({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")), (
({"drop_params": True}, "python-httpx/0.28.1"),
({"drop_params": " true "}, "python-httpx/0.28.1"),
({}, "claude-cli/2.1.0 (external, cli)"),
),
) )
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send( async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send(
monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str