fix(count_tokens): send inference's betas and tools on the Claude count, and retry a rejected count without the forwarded options

The Claude count now builds its anthropic-beta set and tools through the same provider config inference uses, so effort, thinking display, the caller's anthropic-beta header, and Bedrock's tool rewrites match the request inference would send. When the provider still rejects a count that carries the forwarded thinking, tool, output, or context options with a 4xx other than 429, the proxy counts once more without them, which is the request it sent before those options were forwarded. Pass-through Bedrock counts go back to detecting Converse or InvokeModel from the body alone
This commit is contained in:
mateo-berri 2026-11-06 15:09:46 -08:00
parent 8db6f0d5ed
commit f84c244f3e
23 changed files with 654 additions and 177 deletions

View file

@ -51,6 +51,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
count_options: Mapping[str, JsonValue] = MappingProxyType({}), count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
""" """
@ -96,7 +97,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
pass pass
claude_options: Final = claude_count_options( claude_options: Final = claude_count_options(
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, context_management, deployment model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, anthropic_beta, deployment
) )
if isinstance(claude_options, ChatCountTranslationError): if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "anthropic_api") return claude_options.response(request_model, model_to_use, "anthropic_api")
@ -138,7 +139,9 @@ class AnthropicTokenCounter(BaseTokenCounter):
else [dict(message) for message in claude_options.messages], else [dict(message) for message in claude_options.messages],
auth_header=auth_header, auth_header=auth_header,
api_base=api_base, api_base=api_base,
tools=tools, tools=tools
if claude_options.messages is None
else [dict(tool) for tool in claude_options.tools or ()] or None,
system=system, system=system,
optional_params={**count_options, **claude_options.options}, optional_params={**count_options, **claude_options.options},
betas=claude_options.betas, betas=claude_options.betas,

View file

@ -29,7 +29,7 @@ from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
@ -337,31 +337,41 @@ class ClaudeCountOptions:
options: Mapping[str, JsonValue] options: Mapping[str, JsonValue]
betas: tuple[str, ...] betas: tuple[str, ...]
messages: tuple[Mapping[str, JsonValue], ...] | None messages: tuple[Mapping[str, JsonValue], ...] | None
tools: tuple[Mapping[str, JsonValue], ...] | None
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None) NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1 _MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
def _with_provider_betas(headers: Mapping[str, str], provider: str) -> Mapping[str, str]:
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
beta_header: Final = headers.get("anthropic-beta")
if beta_header is None:
return headers
supported: Final = filter_and_transform_beta_headers(
[beta.strip() for beta in beta_header.split(",") if beta.strip()], provider
)
other_headers: Final = {name: value for name, value in headers.items() if name != "anthropic-beta"}
return {**other_headers, "anthropic-beta": ",".join(supported)} if supported else other_headers
def claude_count_options( def claude_count_options(
model: str, model: str,
provider: LlmProviders, provider: LlmProviders,
messages: Sequence[Mapping[str, object]], messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None, tools: Sequence[Mapping[str, object]] | None,
request_options: Mapping[str, JsonValue], request_options: Mapping[str, JsonValue],
context_management: CountContextManagement | None, anthropic_beta: str | None,
deployment: Mapping[str, object] | None, deployment: Mapping[str, object] | None,
) -> ClaudeCountOptions | ChatCountTranslationError: ) -> ClaudeCountOptions | ChatCountTranslationError:
sent: Final = { sent: Final = {name: value for name, value in request_options.items() if value is not None}
**{name: value for name, value in request_options.items() if value is not None},
**({} if context_management is None else {"context_management": context_management.spec}),
}
if not sent: if not sent:
return NO_CLAUDE_COUNT_OPTIONS return NO_CLAUDE_COUNT_OPTIONS
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.exceptions import BadRequestError from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError, context_management_betas from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.types.router import GenericLiteLLMParams from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ProviderConfigManager from litellm.utils import ProviderConfigManager
@ -372,37 +382,39 @@ def claude_count_options(
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
leading_system, conversation = split_leading_system_run(messages) leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_tools: Final = copy.deepcopy([dict(tool) for tool in tools or ()]) sent_params: Final = {
**copy.deepcopy(sent),
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
}
beta_headers: Final = messages_config.headers_with_anthropic_betas(
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
)
sent_headers: Final = dict(
_with_provider_betas(beta_headers, provider.value)
if messages_config.should_filter_anthropic_beta_headers()
else beta_headers
)
try: try:
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model, model=model,
messages=sent_conversation, messages=sent_conversation,
anthropic_messages_optional_request_params={ anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
**copy.deepcopy(sent),
**({"tools": sent_tools} if sent_tools else {}),
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
},
litellm_params=GenericLiteLLMParams.model_validate( litellm_params=GenericLiteLLMParams.model_validate(
{ {
"drop_params": deployment_params.get("drop_params"), "drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"), "model_info": resolved_deployment.get("model_info"),
} }
), ),
headers={}, headers=sent_headers,
) )
except (AnthropicError, BadRequestError) as e: except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message)) return ChatCountTranslationError(message=str(e.message))
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
options: Final = _count_options(transformed)
betas: Final = sorted(context_management_betas(options.get("context_management")))
return ClaudeCountOptions( return ClaudeCountOptions(
options=options, options=_count_options(transformed),
betas=tuple( betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
filter_and_transform_beta_headers(betas, provider.value)
if messages_config.should_filter_anthropic_beta_headers()
else betas
),
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]), messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
tools=transformed.get("tools"),
) )

View file

@ -413,6 +413,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
messages=messages, messages=messages,
) )
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return self._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared beta step takes and returns bare dicts
headers=dict(headers),
optional_params=dict(optional_params),
messages=messages,
)
@staticmethod @staticmethod
def _translate_reasoning_effort_to_anthropic( def _translate_reasoning_effort_to_anthropic(
model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str

View file

@ -65,6 +65,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
count_options: Mapping[str, JsonValue] = MappingProxyType({}), count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
""" """
@ -110,7 +111,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
pass pass
claude_options: Final = claude_count_options( claude_options: Final = claude_count_options(
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, context_management, deployment model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, anthropic_beta, deployment
) )
if isinstance(claude_options, ChatCountTranslationError): if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api") return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
@ -142,7 +143,9 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
api_key=api_key, api_key=api_key,
api_base=api_base, api_base=api_base,
litellm_params=litellm_params, litellm_params=litellm_params,
tools=tools, tools=tools
if claude_options.messages is None
else [dict(tool) for tool in claude_options.tools or ()] or None,
system=system, system=system,
optional_params={**count_options, **claude_options.options}, optional_params={**count_options, **claude_options.options},
betas=claude_options.betas, betas=claude_options.betas,

View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import AsyncIterator, Mapping from collections.abc import AsyncIterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final from typing import TYPE_CHECKING, Any, Final
import httpx import httpx
@ -127,6 +127,18 @@ class BaseAnthropicMessagesConfig(ABC):
""" """
return headers, None return headers, None
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return headers
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
beta_header: Final = next((value for name, value in headers.items() if name.lower() == "anthropic-beta"), "")
return tuple(sorted(frozenset(beta.strip() for beta in beta_header.split(",")) - {""}))
def should_filter_anthropic_beta_headers(self) -> bool: def should_filter_anthropic_beta_headers(self) -> bool:
""" """
Whether ``anthropic-beta`` header values should be filtered down to the Whether ``anthropic-beta`` header values should be filtered down to the

View file

@ -43,6 +43,7 @@ class BaseTokenCounter(ABC):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
pass pass

View file

@ -2,7 +2,7 @@
Bedrock Token Counter implementation using the CountTokens API. Bedrock Token Counter implementation using the CountTokens API.
""" """
from collections.abc import Mapping, Sequence from collections.abc import Awaitable, Mapping, Sequence
from dataclasses import dataclass from dataclasses import dataclass
from types import MappingProxyType from types import MappingProxyType
from typing import Any, Final from typing import Any, Final
@ -117,21 +117,19 @@ class BedrockTokenCounter(BaseTokenCounter):
) -> bool: ) -> bool:
return custom_llm_provider == LlmProviders.BEDROCK.value return custom_llm_provider == LlmProviders.BEDROCK.value
async def _count_with( def _count_on_mantle(
self, self, request_data: Mapping[str, object], litellm_params: Mapping[str, object], resolved_model: str
handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler, ) -> Awaitable[Mapping[str, JsonValue]]:
tokenizer_type: str, return self._mantle_handler.handle_count_tokens_request(
request_data: Mapping[str, object], request_data=dict(request_data),
litellm_params: dict[str, object], litellm_params=dict(litellm_params),
resolved_model: str, resolved_model=resolved_model,
) -> CountTokensOutcome: client=self._client,
)
async def _count_with(self, count: Awaitable[Mapping[str, JsonValue]], tokenizer_type: str) -> CountTokensOutcome:
try: try:
result: Final = await handler.handle_count_tokens_request( result: Final = await count
request_data=dict(request_data),
litellm_params=litellm_params,
resolved_model=resolved_model,
client=self._client,
)
reply: Final = _CountTokensReply.model_validate(result) reply: Final = _CountTokensReply.model_validate(result)
except BedrockError as e: except BedrockError as e:
verbose_logger.debug( verbose_logger.debug(
@ -157,6 +155,7 @@ class BedrockTokenCounter(BaseTokenCounter):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
if not messages: if not messages:
return None return None
@ -182,14 +181,15 @@ class BedrockTokenCounter(BaseTokenCounter):
if isinstance(translated, ChatCountTranslationError): if isinstance(translated, ChatCountTranslationError):
return translated.response(request_model, model_to_use, primary_tokenizer) return translated.response(request_model, model_to_use, primary_tokenizer)
sent_messages: Final = messages if translated is None else translated.messages sent_messages: Final = messages if translated is None else translated.messages
sent_tools: Final = tools if translated is None else translated.tools
claude_options: Final = ( claude_options: Final = (
claude_count_options( claude_count_options(
counted_model, counted_model,
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
sent_messages, sent_messages,
tools if translated is None else translated.tools, sent_tools,
request_options, request_options,
context_management, anthropic_beta,
deployment, deployment,
) )
if is_mantle_claude_model(resolved_model) if is_mantle_claude_model(resolved_model)
@ -197,29 +197,37 @@ class BedrockTokenCounter(BaseTokenCounter):
) )
if isinstance(claude_options, ChatCountTranslationError): if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, primary_tokenizer) return claude_options.response(request_model, model_to_use, primary_tokenizer)
counted_messages: Final = sent_messages if claude_options.messages is None else claude_options.messages invoke_fields: Final = _invoke_fields(claude_options)
runtime_request: Final = { runtime_request: Final = {
**( **_request_data(
_request_data(model_to_use, counted_messages, tools, system) model_to_use,
if translated is None sent_messages if claude_options.messages is None else claude_options.messages,
else _request_data(model_to_use, counted_messages, translated.tools, translated.system) sent_tools if claude_options.messages is None else claude_options.tools,
system if translated is None else translated.system,
), ),
**_invoke_fields(claude_options), **invoke_fields,
} }
mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options} mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
primary: Final = ( primary: Final = (
await self._count_with( await self._count_with(
self._mantle_handler, primary_tokenizer, mantle_request, litellm_params, resolved_model self._count_on_mantle(mantle_request, litellm_params, resolved_model), primary_tokenizer
) )
if use_mantle if use_mantle
else await self._count_with( else await self._count_with(
self._runtime_handler, primary_tokenizer, runtime_request, litellm_params, resolved_model self._runtime_handler.handle_count_tokens_request(
request_data=dict(runtime_request),
litellm_params=litellm_params,
resolved_model=resolved_model,
client=self._client,
input_type="invokeModel" if invoke_fields else None,
),
primary_tokenizer,
) )
) )
outcome: Final = ( outcome: Final = (
await self._count_with( await self._count_with(
self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE
) )
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
else primary else primary

View file

@ -12,7 +12,7 @@ import litellm
from litellm._logging import verbose_logger from litellm._logging import verbose_logger
from litellm.llms.bedrock.base_aws_llm import run_aws_signing from litellm.llms.bedrock.base_aws_llm import run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig, BedrockCountTokensInputType
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
@ -29,6 +29,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
litellm_params: dict[str, Any], litellm_params: dict[str, Any],
resolved_model: str, resolved_model: str,
client: AsyncHTTPHandler | None = None, client: AsyncHTTPHandler | None = None,
input_type: BedrockCountTokensInputType | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
""" """
Handle a CountTokens request using existing LiteLLM patterns. Handle a CountTokens request using existing LiteLLM patterns.
@ -43,7 +44,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
""" """
try: try:
# Validate the request # Validate the request
self.validate_count_tokens_request(request_data) self.validate_count_tokens_request(request_data, input_type)
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model) verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
@ -58,7 +59,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug("Retrieved AWS region: %s", aws_region_name) verbose_logger.debug("Retrieved AWS region: %s", aws_region_name)
# Transform request to Bedrock format (supports both Converse and InvokeModel) # Transform request to Bedrock format (supports both Converse and InvokeModel)
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data) bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(
request_data=request_data, input_type=input_type
)
verbose_logger.debug("Transformed request: %s", bedrock_request) verbose_logger.debug("Transformed request: %s", bedrock_request)

View file

@ -7,7 +7,7 @@ to AWS Bedrock's CountTokens API format and vice versa.
import re import re
from collections.abc import Mapping from collections.abc import Mapping
from typing import Final, Literal from typing import Final, Literal, TypeAlias
from pydantic import JsonValue from pydantic import JsonValue
@ -19,9 +19,7 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
# max_tokens field; CountTokens only counts input, so it has no effect # max_tokens field; CountTokens only counts input, so it has no effect
# on any generation. # on any generation.
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024 DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024
_INVOKE_MODEL_ONLY_FIELDS: Final = frozenset( BedrockCountTokensInputType: TypeAlias = Literal["converse", "invokeModel"]
("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta")
)
def _invoke_model_max_tokens(thinking: JsonValue) -> int: def _invoke_model_max_tokens(thinking: JsonValue) -> int:
@ -86,7 +84,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
- Response: {"inputTokens": <number>} - Response: {"inputTokens": <number>}
""" """
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> Literal["converse", "invokeModel"]: def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> BedrockCountTokensInputType:
""" """
Detect whether to use 'converse' or 'invokeModel' input format. Detect whether to use 'converse' or 'invokeModel' input format.
@ -97,8 +95,6 @@ class BedrockCountTokensConfig(BaseAWSLLM):
'converse' or 'invokeModel' 'converse' or 'invokeModel'
""" """
messages: Final = request_data.get("messages") messages: Final = request_data.get("messages")
if not _INVOKE_MODEL_ONLY_FIELDS.isdisjoint(request_data):
return "invokeModel"
if isinstance(messages, list): if isinstance(messages, list):
# Anthropic content blocks carry a "type" key ({"type": "text", ...}); # Anthropic content blocks carry a "type" key ({"type": "text", ...});
# Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse # Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
@ -119,6 +115,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
def transform_anthropic_to_bedrock_count_tokens( def transform_anthropic_to_bedrock_count_tokens(
self, self,
request_data: Mapping[str, JsonValue], request_data: Mapping[str, JsonValue],
input_type: BedrockCountTokensInputType | None = None,
) -> dict[str, JsonValue]: ) -> dict[str, JsonValue]:
""" """
Transform request to Bedrock CountTokens format. Transform request to Bedrock CountTokens format.
@ -149,9 +146,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
} }
} }
""" """
input_type: Final = self._detect_input_type(request_data) if (input_type or self._detect_input_type(request_data)) == "converse":
if input_type == "converse":
return self._transform_to_converse_format(request_data) return self._transform_to_converse_format(request_data)
else: else:
return self._transform_to_invoke_model_format(request_data) return self._transform_to_invoke_model_format(request_data)
@ -270,23 +265,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
return {"input_tokens": input_tokens} return {"input_tokens": input_tokens}
def validate_count_tokens_request(self, request_data: Mapping[str, JsonValue]) -> None: def validate_count_tokens_request(
self, request_data: Mapping[str, JsonValue], input_type: BedrockCountTokensInputType | None = None
) -> None:
""" """
Validate the incoming count tokens request. Validate the incoming count tokens request.
Supports both Converse and InvokeModel input formats. Supports both Converse and InvokeModel input formats.
Args:
request_data: The request payload
Raises: Raises:
ValueError: If the request is invalid ValueError: If the request is invalid
""" """
if not request_data.get("model"): if not request_data.get("model"):
raise ValueError("model parameter is required") raise ValueError("model parameter is required")
input_type: Final = self._detect_input_type(request_data) if (input_type or self._detect_input_type(request_data)) == "converse":
if input_type == "converse":
# Validate Converse format (messages-based) # Validate Converse format (messages-based)
messages: Final = request_data.get("messages", []) messages: Final = request_data.get("messages", [])
if not messages: if not messages:

View file

@ -76,6 +76,7 @@ else:
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...])
class AmazonAnthropicClaudeMessagesConfig( class AmazonAnthropicClaudeMessagesConfig(
@ -105,6 +106,17 @@ class AmazonAnthropicClaudeMessagesConfig(
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return True return True
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return headers
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ())
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
def get_error_class( def get_error_class(

View file

@ -18,6 +18,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
DEFAULT_ANTHROPIC_API_VERSION, DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig, AnthropicMessagesConfig,
) )
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.llms.bedrock.common_utils import build_mantle_messages_url
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
AmazonAnthropicClaudeMessagesConfig, AmazonAnthropicClaudeMessagesConfig,
@ -70,6 +71,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return False return False
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
return BaseAnthropicMessagesConfig.sent_anthropic_betas(self, headers, request)
def get_complete_url( def get_complete_url(
self, self,
api_base: str | None, api_base: str | None,

View file

@ -535,6 +535,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
import copy import copy

View file

@ -149,6 +149,7 @@ class OpenAITokenCounter(BaseTokenCounter):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
""" """
Count tokens using OpenAI's Responses API /input_tokens endpoint. Count tokens using OpenAI's Responses API /input_tokens endpoint.

View file

@ -1453,6 +1453,7 @@ class VertexAITokenCounter(BaseTokenCounter):
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
count_options: Mapping[str, JsonValue] = MappingProxyType({}), count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
import copy import copy
@ -1519,7 +1520,7 @@ class VertexAITokenCounter(BaseTokenCounter):
sent_messages, sent_messages,
tools, tools,
request_options, request_options,
context_management, anthropic_beta,
deployment, deployment,
) )
if isinstance(claude_options, ChatCountTranslationError): if isinstance(claude_options, ChatCountTranslationError):
@ -1556,7 +1557,9 @@ class VertexAITokenCounter(BaseTokenCounter):
vertex_location=vertex_location, vertex_location=vertex_location,
vertex_credentials=vertex_credentials, vertex_credentials=vertex_credentials,
system=system, system=system,
tools=tools, tools=tools
if claude_options.messages is None
else [dict(tool) for tool in claude_options.tools or ()] or None,
count_options={**count_options, **claude_options.options}, count_options={**count_options, **claude_options.options},
betas=claude_options.betas, betas=claude_options.betas,
) )

View file

@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence
from typing import Final from typing import Final
from litellm.exceptions import UnsupportedParamsError from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.messages.native.transformation import ( from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig, AnthropicMessagesConfig,
@ -51,6 +51,11 @@ def _vertex_version(headers: Mapping[str, object], model: str, drop_params: bool
) )
def _is_web_search(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return isinstance(tool_type, str) and tool_type.startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value)
def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers
return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY} return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY}
@ -94,10 +99,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
project_id=vertex_ai_project, project_id=vertex_ai_project,
custom_llm_provider="vertex_ai", custom_llm_provider="vertex_ai",
) )
return self._finalize_vertex_messages_environment( vertex_headers, vertex_api_base = self._finalize_vertex_messages_environment(
headers, headers,
model, model,
messages,
optional_params, optional_params,
api_base, api_base,
access_token, access_token,
@ -105,6 +109,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
vertex_ai_project, vertex_ai_project,
vertex_ai_location, vertex_ai_location,
) )
return dict(self.headers_with_anthropic_betas(vertex_headers, optional_params, messages)), vertex_api_base
async def avalidate_anthropic_messages_environment( async def avalidate_anthropic_messages_environment(
self, self,
@ -122,10 +127,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
project_id=vertex_ai_project, project_id=vertex_ai_project,
custom_llm_provider="vertex_ai", custom_llm_provider="vertex_ai",
) )
return self._finalize_vertex_messages_environment( vertex_headers, vertex_api_base = self._finalize_vertex_messages_environment(
headers, headers,
model, model,
messages,
optional_params, optional_params,
api_base, api_base,
access_token, access_token,
@ -133,12 +137,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
vertex_ai_project, vertex_ai_project,
vertex_ai_location, vertex_ai_location,
) )
return dict(self.headers_with_anthropic_betas(vertex_headers, optional_params, messages)), vertex_api_base
def _finalize_vertex_messages_environment( def _finalize_vertex_messages_environment(
self, self,
headers: Mapping[str, str], headers: Mapping[str, str],
model: str, model: str,
messages: Sequence[object],
optional_params: dict, # mutable-ok: preserves the existing provider optional-parameter contract optional_params: dict, # mutable-ok: preserves the existing provider optional-parameter contract
api_base: str | None, api_base: str | None,
access_token: str, access_token: str,
@ -162,39 +166,48 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
request_headers["content-type"] = "application/json" request_headers["content-type"] = "application/json"
# Add beta headers for Vertex AI
tools: Final = optional_params.get("tools", [])
beta_values: Final[set[str]] = set()
# Get existing beta headers if any
existing_beta: Final = request_headers.get("anthropic-beta")
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
# Check for web search tool
for tool in tools:
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value)
break
# Check for tool search tools - Vertex AI uses different beta header
anthropic_model_info: Final = AnthropicModelInfo()
if anthropic_model_info.is_tool_search_used(tools):
beta_values.add(get_tool_search_beta_header("vertex_ai"))
if optional_params.get("safeguards") is not None:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
if beta_values:
request_headers["anthropic-beta"] = ",".join(beta_values)
return request_headers, api_base return request_headers, api_base
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
tools: Final = tuple(
tool for tool in map(as_str_mapping, iter_items(optional_params.get("tools") or ())) if tool is not None
)
existing_betas: Final = (headers.get("anthropic-beta") or "").split(",")
beta_values: Final = frozenset(
(
*(beta.strip() for beta in existing_betas if beta.strip()),
*context_management_betas(optional_params.get("context_management")), # pyright: ignore[reportArgumentType] # optional_params is the config contract's untyped request body
*(
(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value,)
if any(map(_is_web_search, tools))
else ()
),
*(
(get_tool_search_beta_header("vertex_ai"),)
if AnthropicModelInfo().is_tool_search_used(list(tools))
else ()
),
*(
(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,)
if optional_params.get("safeguards") is not None
else ()
),
*(
(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value,)
if messages_carry_output_config(messages)
else ()
),
)
)
if not beta_values:
return headers
return {**headers, "anthropic-beta": ",".join(sorted(beta_values))}
def get_complete_url( def get_complete_url(
self, self,
api_base: str | None, api_base: str | None,

View file

@ -336,7 +336,7 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params") _COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format") _COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format", "context_management")
class _CountTokensBodyOptions(TypedDict): class _CountTokensBodyOptions(TypedDict):
@ -429,6 +429,7 @@ async def count_tokens(
messages_format="anthropic", messages_format="anthropic",
litellm_call_id=litellm_call_id, litellm_call_id=litellm_call_id,
client_oauth_token=forwarded_anthropic_oauth_token(request.headers), client_oauth_token=forwarded_anthropic_oauth_token(request.headers),
anthropic_beta=request.headers.get("anthropic-beta"),
**_count_tokens_body_options(data, auto_drop_params), **_count_tokens_body_options(data, auto_drop_params),
) )
except HTTPException as e: except HTTPException as e:

View file

@ -72,6 +72,10 @@ class ProviderCountFailure:
def denied_in_region(self) -> bool: def denied_in_region(self) -> bool:
return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES
@property
def rejected_request(self) -> bool:
return 400 <= self.status_code < 500 and self.status_code != 429
def with_hint(self, hint: str) -> "ProviderCountFailure": def with_hint(self, hint: str) -> "ProviderCountFailure":
if self.rejected_before_provider_call: if self.rejected_before_provider_call:
return self return self

View file

@ -14813,6 +14813,7 @@ async def _try_provider_token_count(
context_management: CountContextManagement | None = None, context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]: ) -> Union["TokenCountResponse", ProviderCountFailure, None]:
"""Attempt provider-specific token counting. Returns None to fall through to local counting.""" """Attempt provider-specific token counting. Returns None to fall through to local counting."""
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
@ -14841,6 +14842,7 @@ async def _try_provider_token_count(
context_management=context_management, context_management=context_management,
chat_params=chat_params, chat_params=chat_params,
request_options=request_options, request_options=request_options,
anthropic_beta=anthropic_beta,
) )
except httpx.HTTPStatusError as e: except httpx.HTTPStatusError as e:
return ProviderCountFailure( return ProviderCountFailure(
@ -14923,6 +14925,32 @@ async def _provider_token_count(
return failure return failure
def _retried_without_rejected_options(
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
request_options: Mapping[str, JsonValue],
) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]:
if not request_options:
return count_with
return partial(_count_retrying_without_options, count_with)
async def _count_retrying_without_options(
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
model: str,
region: str | None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
counted: Final = await count_with(model, region)
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
return counted
verbose_proxy_logger.warning(
"Token count for %s answered %s with the request's thinking, tool, and output options, "
"so it counts again without them",
model,
counted.status_code,
)
return await count_with(model, region, request_options=MappingProxyType({}))
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class _LocalTokenEstimate: class _LocalTokenEstimate:
total_tokens: int total_tokens: int
@ -15335,6 +15363,7 @@ async def count_request_tokens(
litellm_call_id: str | None = None, litellm_call_id: str | None = None,
client_oauth_token: str | None = None, client_oauth_token: str | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse: ) -> TokenCountResponse:
call_id: Final = resolve_litellm_call_id(litellm_call_id) call_id: Final = resolve_litellm_call_id(litellm_call_id)
if request.prompt is None and request.messages is None and request.contents is None: if request.prompt is None and request.messages is None and request.contents is None:
@ -15355,6 +15384,7 @@ async def count_request_tokens(
responses_request=responses_request, responses_request=responses_request,
client_oauth_token=client_oauth_token, client_oauth_token=client_oauth_token,
request_options=request_options, request_options=request_options,
anthropic_beta=anthropic_beta,
) )
return await logged_token_count( return await logged_token_count(
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count, _with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
@ -15398,6 +15428,7 @@ async def _count_request_tokens(
responses_request: Mapping[str, object] | None, responses_request: Mapping[str, object] | None,
client_oauth_token: str | None, client_oauth_token: str | None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse: ) -> TokenCountResponse:
prompt: Final = request.prompt prompt: Final = request.prompt
messages: Final = request.messages messages: Final = request.messages
@ -15474,6 +15505,7 @@ async def _count_request_tokens(
) )
if isinstance(chat_count, ChatCountBridgeError): if isinstance(chat_count, ChatCountBridgeError):
raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400) raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400)
counted_options: Final = _count_request_options(request_options, count_deployment, sent_params)
ladder: Final = ( ladder: Final = (
None None
if provider_counter is None if provider_counter is None
@ -15485,27 +15517,33 @@ async def _count_request_tokens(
routed_deployment, routed_deployment,
request.model, request.model,
) )
or partial( or _retried_without_rejected_options(
_try_provider_token_count, partial(
provider_counter, _try_provider_token_count,
custom_llm_provider, provider_counter,
messages=( custom_llm_provider,
[dict(message) for message in chat_count.messages] messages=(
if chat_count is not None [dict(message) for message in chat_count.messages]
else [{"role": "user", "content": prompt}] if chat_count is not None
if prompt_as_message else [{"role": "user", "content": prompt}]
else request_messages if prompt_as_message
else request_messages
),
contents=None if prompt_as_message else contents,
deployment=count_deployment,
request_model=request.model,
tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None,
system=system,
messages_format=messages_format,
output_format=output_format,
context_management=_count_context_management(
context_management, request.model, request_litellm_params
),
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
request_options=counted_options,
anthropic_beta=anthropic_beta,
), ),
contents=None if prompt_as_message else contents, counted_options,
deployment=count_deployment,
request_model=request.model,
tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None,
system=system,
messages_format=messages_format,
output_format=output_format,
context_management=_count_context_management(context_management, request.model, request_litellm_params),
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
request_options=_count_request_options(request_options, count_deployment, sent_params),
), ),
model_to_use=model_to_use, model_to_use=model_to_use,
request_model=request.model, request_model=request.model,

View file

@ -27,8 +27,12 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig,
) )
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES from litellm.types.llms.anthropic import (
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
)
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -488,7 +492,7 @@ async def _counted_requests(
messages_format: TokenCountMessagesFormat, messages_format: TokenCountMessagesFormat,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}), chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}),
context_management: CountContextManagement | None = None, anthropic_beta: str | None = None,
model: str = "claude-opus-5-5", model: str = "claude-opus-5-5",
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: ) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
@ -509,7 +513,7 @@ async def _counted_requests(
messages_format=messages_format, messages_format=messages_format,
chat_params=chat_params, chat_params=chat_params,
request_options=request_options, request_options=request_options,
context_management=context_management, anthropic_beta=anthropic_beta,
) )
return result, tuple(sent.get() for _ in range(sent.qsize())) return result, tuple(sent.get() for _ in range(sent.qsize()))
@ -880,12 +884,6 @@ _MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
_INFERENCE_PROVIDERS: Final = MappingProxyType( _INFERENCE_PROVIDERS: Final = MappingProxyType(
{AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"} {AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"}
) )
_CONTEXT_BETAS: Final = frozenset(
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
)
)
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA} _WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}
_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000} _ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000}
@ -894,6 +892,8 @@ async def _messages_inference_request(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
inference_endpoint: str, inference_endpoint: str,
request_options: Mapping[str, JsonValue], request_options: Mapping[str, JsonValue],
model: str = "claude-haiku-5-5",
extra_headers: Mapping[str, str] = MappingProxyType({}),
) -> httpx.Request: ) -> httpx.Request:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
@ -904,12 +904,13 @@ async def _messages_inference_request(
with respx.mock: with respx.mock:
respx.post(inference_endpoint).mock(side_effect=reply) respx.post(inference_endpoint).mock(side_effect=reply)
await litellm.anthropic_messages( await litellm.anthropic_messages(
model=f"{_INFERENCE_PROVIDERS[counter_type]}/claude-haiku-5-5", model=f"{_INFERENCE_PROVIDERS[counter_type]}/{model}",
messages=[{"role": "user", "content": "Weather in SF?"}], messages=[{"role": "user", "content": "Weather in SF?"}],
tools=[_WEATHER_TOOL], tools=[_WEATHER_TOOL],
max_tokens=4096, max_tokens=4096,
api_key="sk-ant-api03-test-key", api_key="sk-ant-api03-test-key",
api_base="https://gateway.example", api_base="https://gateway.example",
extra_headers=dict(extra_headers),
**request_options, **request_options,
) )
return sent.get() return sent.get()
@ -923,16 +924,24 @@ def _betas(request: httpx.Request) -> frozenset[str]:
@pytest.mark.usefixtures("httpx_transport_clients") @pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS @_COUNTERS
@pytest.mark.parametrize( @pytest.mark.parametrize(
("request_options", "context_management"), ("request_options", "expected_betas"),
( (
({"thinking": _ENABLED_THINKING}, None), ({"thinking": _ENABLED_THINKING}, frozenset()),
({"tool_choice": {"type": "any"}}, None), ({"tool_choice": {"type": "any"}}, frozenset()),
({"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, None),
( (
{"thinking": _ENABLED_THINKING}, {"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}},
{"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]}, frozenset((ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value,)),
), ),
({}, {"edits": [{"type": "compact_20260112"}]}), (
{
"thinking": _ENABLED_THINKING,
"context_management": {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
},
},
frozenset((ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,)),
),
({"context_management": {"edits": [{"type": "compact_20260112"}]}}, frozenset()),
), ),
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"), ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
) )
@ -940,12 +949,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str, count_endpoint: str,
request_options: dict[str, JsonValue], request_options: dict[str, JsonValue],
context_management: dict[str, JsonValue] | None, expected_betas: frozenset[str],
) -> None: ) -> None:
inference: Final = await _messages_inference_request( inference: Final = await _messages_inference_request(
counter_type, counter_type, count_endpoint.removesuffix("/count_tokens"), request_options
count_endpoint.removesuffix("/count_tokens"),
{**request_options, **({} if context_management is None else {"context_management": context_management})},
) )
_, (count,) = await _counted_requests( _, (count,) = await _counted_requests(
counter_type, counter_type,
@ -954,9 +961,6 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
[_WEATHER_TOOL], [_WEATHER_TOOL],
"anthropic", "anthropic",
request_options=request_options, request_options=request_options,
context_management=(
None if context_management is None else CountContextManagement(spec=context_management, drop_invalid_edits=False)
),
model="claude-haiku-5-5", model="claude-haiku-5-5",
) )
@ -973,7 +977,38 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
} }
assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS assert expected_betas <= _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_sends_the_thinking_display_and_caller_betas_messages_inference_sends(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
caller_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
display_thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
inference: Final = await _messages_inference_request(
counter_type,
count_endpoint.removesuffix("/count_tokens"),
display_thinking,
model="claude-opus-4-6",
extra_headers={"anthropic-beta": caller_beta},
)
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
request_options=display_thinking,
anthropic_beta=caller_beta,
model="claude-opus-4-6",
)
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
assert counted_betas == _betas(inference)
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
@pytest.mark.asyncio @pytest.mark.asyncio

View file

@ -1,5 +1,8 @@
import base64 import base64
import json import json
from typing import Final
import pytest
from litellm.llms.bedrock.count_tokens.transformation import ( from litellm.llms.bedrock.count_tokens.transformation import (
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS,
@ -53,6 +56,28 @@ def test_detect_input_type_converse_blocks_route_to_converse():
assert config._detect_input_type(request) == "converse" assert config._detect_input_type(request) == "converse"
@pytest.mark.parametrize(
"option",
(
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
{"tool_choice": {"type": "auto"}},
{"output_config": {"effort": "low"}},
),
)
def test_pass_through_body_with_string_content_is_counted_as_converse_whatever_options_it_carries(
option: dict[str, object],
) -> None:
request: Final = {
"model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
"messages": [{"role": "user", "content": "hi"}],
**option,
}
result: Final = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request)
assert result["input"] == {"converse": {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}}
def test_transform_to_invoke_model_format_base64_encodes_body(): def test_transform_to_invoke_model_format_base64_encodes_body():
"""The CountTokens API expects invokeModel.body as a base64-encoded blob; """The CountTokens API expects invokeModel.body as a base64-encoded blob;
Anthropic Messages bodies additionally need anthropic_version/max_tokens Anthropic Messages bodies additionally need anthropic_version/max_tokens

View file

@ -10,7 +10,10 @@ from pydantic import JsonValue, TypeAdapter
import litellm import litellm
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import CountContextManagement from litellm.types.llms.anthropic import (
ANTHROPIC_EFFORT_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
)
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
@ -350,29 +353,36 @@ _THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}} _WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
async def _messages_inference_and_count(deployment_model: str, model_to_use: str) -> tuple[_Bedrock, _Bedrock]: _CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
async def _messages_inference_and_count(
deployment_model: str,
model_to_use: str,
request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS,
tool: Mapping[str, JsonValue] = _WEATHER_TOOL,
runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}),
) -> tuple[_Bedrock, _Bedrock]:
credentials: Final = DEPLOYMENT["litellm_params"] credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages( await litellm.anthropic_messages(
model=deployment_model, model=deployment_model,
messages=MESSAGES, messages=MESSAGES,
tools=[_WEATHER_TOOL], tools=[dict(tool)],
max_tokens=4096, max_tokens=4096,
context_management=_CONTEXT_EDITS,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**_THINKING_AND_TOOL_CHOICE, **request_options,
**credentials, **credentials,
) )
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens( result: Final = await _counter(bedrock).count_tokens(
model_to_use=model_to_use, model_to_use=model_to_use,
messages=MESSAGES, messages=MESSAGES,
contents=None, contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}}, deployment={"litellm_params": {**credentials, "model": deployment_model}},
request_model="claude", request_model="claude",
tools=[_WEATHER_TOOL], tools=[dict(tool)],
request_options=_THINKING_AND_TOOL_CHOICE, request_options=request_options,
context_management=CountContextManagement(spec=_CONTEXT_EDITS, drop_invalid_edits=False),
) )
assert result is not None assert result is not None
assert result.error is False assert result.error is False
@ -419,6 +429,79 @@ async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_infe
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"] assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
_DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
return tuple(sorted(request.headers["anthropic-beta"].split(",")))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "request_options", "expected_beta"),
(
(
"bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
{"output_config": {"effort": "low"}},
ANTHROPIC_EFFORT_BETA_HEADER,
),
("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER),
),
ids=("effort", "thinking_display"),
)
async def test_runtime_count_sends_the_betas_invoke_inference_sends(
deployment_model: str, request_options: dict[str, JsonValue], expected_beta: str
) -> None:
inference, bedrock = await _messages_inference_and_count(
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta")
assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
assert isinstance(counted_betas, list) and expected_beta in counted_betas
@pytest.mark.asyncio
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/invoke/us.anthropic.claude-opus-5-5",
"us.anthropic.claude-opus-5-5",
_DISPLAY_THINKING,
runtime=(400, UNSUPPORTED),
)
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
assert isinstance(sent_betas, list)
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas))
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in sent_betas
@pytest.mark.asyncio
async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
assert _sent_betas(bedrock.requests[0]) == _sent_betas(inference.requests[0])
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0])
@pytest.mark.asyncio
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/invoke/us.anthropic.claude-opus-5-5",
"us.anthropic.claude-opus-5-5",
_THINKING_AND_TOOL_CHOICE,
tool={**_WEATHER_TOOL, "eager_input_streaming": True},
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
_CITY_SCHEMA: Final[dict[str, JsonValue]] = { _CITY_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object", "type": "object",
"properties": {"city": {"type": "string"}}, "properties": {"city": {"type": "string"}},

View file

@ -2016,8 +2016,7 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
contents=None, contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=tools, tools=tools,
request_options=options, request_options={**options, "context_management": context_edits},
context_management=CountContextManagement(spec=context_edits, drop_invalid_edits=False),
) )
assert result is not None assert result is not None

View file

@ -56,6 +56,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import (
kimi_k3_tokens, kimi_k3_tokens,
load_kimi_k3_encoding, load_kimi_k3_encoding,
) )
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensInputType
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http
@ -368,6 +369,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
litellm_call_id=None, litellm_call_id=None,
client_oauth_token=None, client_oauth_token=None,
request_options=None, request_options=None,
anthropic_beta=None,
): ):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "claude-3-sonnet-20240229" assert request.model == "claude-3-sonnet-20240229"
@ -451,6 +453,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
litellm_call_id=None, litellm_call_id=None,
client_oauth_token=None, client_oauth_token=None,
request_options=None, request_options=None,
anthropic_beta=None,
): ):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "gpt-4" assert request.model == "gpt-4"
@ -940,6 +943,7 @@ class _FailingRuntimeHandler(BedrockCountTokensHandler):
litellm_params: dict[str, object], litellm_params: dict[str, object],
resolved_model: str, resolved_model: str,
client: AsyncHTTPHandler | None = None, client: AsyncHTTPHandler | None = None,
input_type: BedrockCountTokensInputType | None = None,
) -> dict[str, JsonValue]: ) -> dict[str, JsonValue]:
raise self._error raise self._error
@ -1272,6 +1276,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
litellm_call_id=None, litellm_call_id=None,
client_oauth_token=None, client_oauth_token=None,
request_options=None, request_options=None,
anthropic_beta=None,
): ):
raise ProxyException( raise ProxyException(
message='{"detail":{"message":"Input is too long for requested model."}}', message='{"detail":{"message":"Input is too long for requested model."}}',
@ -1332,6 +1337,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
litellm_call_id=None, litellm_call_id=None,
client_oauth_token=None, client_oauth_token=None,
request_options=None, request_options=None,
anthropic_beta=None,
): ):
raise ProxyException( raise ProxyException(
message='{"Message":"Bearer Token has expired"}', message='{"Message":"Bearer Token has expired"}',
@ -1392,6 +1398,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
litellm_call_id=None, litellm_call_id=None,
client_oauth_token=None, client_oauth_token=None,
request_options=None, request_options=None,
anthropic_beta=None,
): ):
raise ProxyException( raise ProxyException(
message="Rate limit exceeded", message="Rate limit exceeded",
@ -1416,7 +1423,9 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
proxy_server.count_request_tokens = original_token_counter proxy_server.count_request_tokens = original_token_counter
def _json_request(body: Mapping[str, object], user_agent: str) -> Request: def _json_request(
body: Mapping[str, object], user_agent: str, headers: Mapping[str, str] = MappingProxyType({})
) -> Request:
payload: Final = json.dumps(body).encode() payload: Final = json.dumps(body).encode()
async def receive() -> Message: async def receive() -> Message:
@ -1428,7 +1437,11 @@ def _json_request(body: Mapping[str, object], user_agent: str) -> Request:
"method": "POST", "method": "POST",
"path": "/v1/messages/count_tokens", "path": "/v1/messages/count_tokens",
"query_string": b"", "query_string": b"",
"headers": [(b"content-type", b"application/json"), (b"user-agent", user_agent.encode())], "headers": [
(b"content-type", b"application/json"),
(b"user-agent", user_agent.encode()),
*((name.encode(), value.encode()) for name, value in headers.items()),
],
}, },
receive, receive,
) )
@ -1641,10 +1654,11 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize( @pytest.mark.parametrize(
("additional_drop_params", "expected_paths"), ("additional_drop_params", "request_params", "expected_paths"),
( (
( (
None, None,
{},
{ {
"thinking", "thinking",
"tool_choice", "tool_choice",
@ -1656,17 +1670,32 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
), ),
( (
["tool_choice", "output_config.format"], ["tool_choice", "output_config.format"],
{},
{"thinking", "output_config", "output_config.effort", "context_management"}, {"thinking", "output_config", "output_config.effort", "context_management"},
), ),
( (
["context_management"], ["context_management"],
{},
{"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"}, {"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"},
), ),
(
None,
{"additional_drop_params": ["thinking", "context_management"]},
{"tool_choice", "output_config", "output_config.effort", "output_config.format"},
),
),
ids=(
"nothing-dropped",
"deployment-drops-tool-choice-and-the-format",
"deployment-drops-context-management",
"request-drops-thinking-and-context-management",
), ),
ids=("nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management"),
) )
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends( async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
monkeypatch: pytest.MonkeyPatch, additional_drop_params: list[str] | None, expected_paths: set[str] monkeypatch: pytest.MonkeyPatch,
additional_drop_params: list[str] | None,
request_params: dict[str, JsonValue],
expected_paths: set[str],
) -> None: ) -> None:
import litellm.proxy.proxy_server as proxy_server import litellm.proxy.proxy_server as proxy_server
@ -1688,6 +1717,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
"messages": [{"role": "user", "content": "Weather in SF?"}], "messages": [{"role": "user", "content": "Weather in SF?"}],
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
**_CLAUDE_COUNT_OPTIONS, **_CLAUDE_COUNT_OPTIONS,
**request_params,
}, },
"python-httpx/0.28.1", "python-httpx/0.28.1",
) )
@ -1709,6 +1739,187 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
) )
_CALLER_BETA: Final = "web-fetch-2025-09-10"
async def _count_on_a_claude_deployment(
monkeypatch: pytest.MonkeyPatch,
deployment_model: str,
body: Mapping[str, JsonValue],
reply: Callable[[httpx.Request], httpx.Response],
user_agent: str = "python-httpx/0.28.1",
) -> tuple[object, tuple[httpx.Request, ...]]:
import litellm.proxy.proxy_server as proxy_server
router: Final = Router(
model_list=[
{"model_name": "claude-agent", "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test"}}
]
)
request: Final = _json_request(
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
user_agent,
{"anthropic-beta": _CALLER_BETA},
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(side_effect=reply)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
return response, tuple(call.request for call in route.calls)
def _counts_only_without_options(request: httpx.Request) -> httpx.Response:
if _sent_option_paths(json.loads(request.content)):
return httpx.Response(
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}}
)
return httpx.Response(200, json={"input_tokens": 7})
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "options", "rejected_by_anthropic"),
(
("anthropic/claude-haiku-4-5", {"output_config": {"effort": "fastest"}}, False),
("anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, True),
),
ids=("rejected-by-the-count-transform", "rejected-by-anthropic"),
)
async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_claude_count_rejects_them(
monkeypatch: pytest.MonkeyPatch,
deployment_model: str,
options: dict[str, JsonValue],
rejected_by_anthropic: bool,
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch, deployment_model, options, _counts_only_without_options
)
assert not isinstance(response, JSONResponse)
assert response["input_tokens"] == 7
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
*((True,) if rejected_by_anthropic else ()),
False,
]
assert [_CALLER_BETA in sent.headers.get("anthropic-beta", "") for sent in requests] == [
*((True,) if rejected_by_anthropic else ()),
False,
]
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code", (401, 403, 404))
async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_options_also_gets(
monkeypatch: pytest.MonkeyPatch, status_code: int
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-haiku-5-5",
_CLAUDE_COUNT_OPTIONS,
lambda _: httpx.Response(status_code, json={"type": "error", "error": {"type": "error", "message": "denied"}}),
)
assert isinstance(response, JSONResponse)
assert response.status_code == status_code
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [True, False]
@pytest.mark.asyncio
async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-haiku-5-5",
_CLAUDE_COUNT_OPTIONS,
lambda _: httpx.Response(429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow"}}),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 429
assert len(requests) == 1
async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
},
)
)
await litellm.anthropic_messages(
model=f"anthropic/{model}",
messages=[{"role": "user", "content": "Weather in SF?"}],
max_tokens=4096,
api_key="sk-ant-test",
**options,
)
return json.loads(route.calls.last.request.content)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_drop_params", "user_agent"),
(({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")),
)
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send(
monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True})
kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking})
assert dropped["thinking"] != kept["thinking"]
response, (count,) = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-opus-4-7",
{"thinking": thinking, **request_drop_params},
lambda _: httpx.Response(200, json={"input_tokens": 7}),
user_agent,
)
assert not isinstance(response, JSONResponse)
counted: Final = json.loads(count.content)
assert {name: counted.get(name) for name in ("thinking", "output_config")} == {
name: dropped.get(name) for name in ("thinking", "output_config")
}
@pytest.mark.asyncio
async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format(
monkeypatch: pytest.MonkeyPatch,
) -> None:
output_format: Final[dict[str, JsonValue]] = {
"type": "json_schema",
"schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
}
response, (count,) = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-haiku-5-5",
{"output_format": output_format},
lambda _: httpx.Response(200, json={"input_tokens": 7}),
)
assert not isinstance(response, JSONResponse)
counted: Final = json.loads(count.content)
assert "output_format" not in counted
assert counted["output_config"]["format"] == output_format
class _ScriptedCounter(BaseTokenCounter): class _ScriptedCounter(BaseTokenCounter):
def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None:
self._replies = replies self._replies = replies
@ -1734,6 +1945,7 @@ class _ScriptedCounter(BaseTokenCounter):
context_management: object = None, context_management: object = None,
chat_params: object = None, chat_params: object = None,
request_options: object = None, request_options: object = None,
anthropic_beta: object = None,
) -> TokenCountResponse | None: ) -> TokenCountResponse | None:
self.calls.append((model_to_use, messages, tools, system)) self.calls.append((model_to_use, messages, tools, system))
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name")) self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))