fix(count_tokens): forward context_management, thinking, tool_choice and output_config to the Claude count APIs

The Anthropic-format count on /v1/messages/count_tokens dropped these fields on anthropic,
azure_ai, vertex_ai and bedrock (runtime and Mantle), so the count disagreed with what
inference sends. The count now runs the deployment's own /v1/messages transform and sends
the same anthropic-beta values as inference, from one shared context_management_betas helper
This commit is contained in:
mateo-berri 2026-10-29 04:02:06 -07:00
parent b7da9a2755
commit f012c60033
23 changed files with 730 additions and 101 deletions

View file

@ -12,7 +12,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -45,6 +45,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -347,6 +348,20 @@ def _beta_header_values(side: str | Sequence[str] | None) -> tuple[str, ...]:
return tuple(entry for entry in side if isinstance(entry, str))
def context_management_betas(context_management: JsonValue) -> frozenset[str]:
"""The anthropic-beta values a ``context_management`` body needs: the compact beta for a
``compact_20260112`` edit and the context-management beta for every other edit."""
edits: Final = context_management.get("edits") if isinstance(context_management, dict) else None
if not isinstance(edits, list):
return frozenset()
return frozenset(
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
if isinstance(edit, dict) and edit.get("type") == "compact_20260112"
else ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
for edit in edits
)
def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta: str | Sequence[str] | None) -> str:
"""Merge anthropic-beta header values, deduplicated and sorted.
@ -1306,8 +1321,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

View file

@ -4,7 +4,7 @@ Anthropic CountTokens API handler.
Uses httpx for HTTP requests instead of the Anthropic SDK.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
betas: Sequence[str] = (),
) -> dict[str, JsonValue]:
"""
Handle a CountTokens request using httpx.
@ -79,7 +80,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, betas)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

View file

@ -16,6 +16,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
anthropic_count_fields,
claude_count_options,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
@ -49,6 +50,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
"""
@ -93,6 +95,12 @@ class AnthropicTokenCounter(BaseTokenCounter):
case None:
pass
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.ANTHROPIC, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "anthropic_api")
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
api_base: Final = litellm_params.get("api_base")
@ -130,7 +138,8 @@ class AnthropicTokenCounter(BaseTokenCounter):
api_base=api_base,
tools=tools,
system=system,
optional_params=count_options,
optional_params={**count_options, **claude_options.options},
betas=claude_options.betas,
)
if result is not None:

View file

@ -29,12 +29,12 @@ from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config", "context_management")
_PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"})
@ -113,17 +113,24 @@ class ChatCountTranslationError:
@with_config(ConfigDict(extra="ignore"))
class _AnthropicChatRequest(TypedDict):
messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]]
system: NotRequired[ReadOnly[JsonValue]]
tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]]
class _CountedOptions(TypedDict):
thinking: NotRequired[ReadOnly[JsonValue]]
tool_choice: NotRequired[ReadOnly[JsonValue]]
output_config: NotRequired[ReadOnly[JsonValue]]
output_format: NotRequired[ReadOnly[JsonValue]]
context_management: NotRequired[ReadOnly[JsonValue]]
@with_config(ConfigDict(extra="ignore"))
class _AnthropicChatRequest(_CountedOptions):
messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]]
system: NotRequired[ReadOnly[JsonValue]]
tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]]
_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest)
_COUNTED_OPTIONS: Final = TypeAdapter(_CountedOptions)
_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object])
_ChatParams: TypeAlias = dict[str, object]
_ChatMessages: TypeAlias = list[AllMessageValues]
@ -240,7 +247,7 @@ def _system_messages(system: object) -> tuple[Mapping[str, object], ...]:
return (message,) if isinstance(system, (str, list)) and system else ()
def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]:
def _count_options(request: _CountedOptions) -> Mapping[str, JsonValue]:
"""Structured output reaches chat inference as the beta ``output_format``, which count_tokens
refuses; its GA ``output_config.format`` form counts the same schema the same way."""
output_config: Final = request.get("output_config")
@ -256,6 +263,7 @@ def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]:
("thinking", request.get("thinking")),
("tool_choice", request.get("tool_choice")),
("output_config", counted_config or None),
("context_management", request.get("context_management")),
)
if value is not None
}
@ -325,6 +333,77 @@ def anthropic_fields_for_chat_tool_use(
)
@dataclass(frozen=True, slots=True)
class ClaudeCountOptions:
options: Mapping[str, JsonValue]
betas: tuple[str, ...]
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=())
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
def claude_count_options(
model: str,
provider: LlmProviders,
request_options: Mapping[str, JsonValue],
context_management: CountContextManagement | None,
deployment: Mapping[str, object] | None,
) -> ClaudeCountOptions | ChatCountTranslationError:
"""The thinking, tool_choice, output_config and context_management a Claude count sends, with the
anthropic-beta values they need, after the deployment's own /v1/messages transform rewrites them.
The transform runs without messages and with a max_tokens above any thinking budget, so it never
caps a budget the caller's real max_tokens would allow. An error it raises is the error inference
answers for the same body.
"""
sent: Final = {
**{name: value for name, value in request_options.items() if value is not None},
**({} if context_management is None else {"context_management": context_management.spec}),
}
if not sent:
return NO_CLAUDE_COUNT_OPTIONS
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError, context_management_betas
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ProviderConfigManager
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
if messages_config is None:
return NO_CLAUDE_COUNT_OPTIONS
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
try:
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
messages=[],
anthropic_messages_optional_request_params={
**copy.deepcopy(sent),
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
},
litellm_params=GenericLiteLLMParams.model_validate(
{
"drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"),
}
),
headers={},
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
options: Final = _count_options(_COUNTED_OPTIONS.validate_python(request))
betas: Final = sorted(context_management_betas(options.get("context_management")))
return ClaudeCountOptions(
options=options,
betas=tuple(
filter_and_transform_beta_headers(betas, provider.value)
if messages_config.should_filter_anthropic_beta_headers()
else betas
),
)
async def anthropic_count_fields(
model: str,
messages: Sequence[Mapping[str, object]],
@ -402,7 +481,7 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(self, auth_header: Mapping[str, str], betas: Sequence[str] = ()) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
@ -412,7 +491,7 @@ class AnthropicCountTokensConfig:
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
auth_header.get("anthropic-beta"), (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *betas)
),
}

View file

@ -31,6 +31,7 @@ from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
AnthropicError,
AnthropicModelInfo,
context_management_betas,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
@ -792,28 +793,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

View file

@ -4,7 +4,7 @@ Azure AI Anthropic CountTokens API handler.
Uses httpx for HTTP requests with Azure authentication.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
@ -13,7 +13,7 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.common_utils import AnthropicError, merge_anthropic_beta_headers
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
@ -38,6 +38,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: object = None,
optional_params: Mapping[str, JsonValue] | None = None,
betas: Sequence[str] = (),
) -> dict[str, object]:
"""
Handle a CountTokens request using httpx with Azure authentication.
@ -101,7 +102,10 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
response: Final = await async_client.post(
endpoint_url,
headers=headers,
headers={
**headers,
"anthropic-beta": merge_anthropic_beta_headers(headers.get("anthropic-beta"), betas),
},
json=request_body,
timeout=request_timeout,
)

View file

@ -15,6 +15,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
anthropic_count_fields,
claude_count_options,
)
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
@ -63,6 +64,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
"""
@ -107,6 +109,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
case None:
pass
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.AZURE_AI, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
@ -134,7 +142,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
litellm_params=litellm_params,
tools=tools,
system=system,
optional_params=count_options,
optional_params={**count_options, **claude_options.options},
betas=claude_options.betas,
)
if result is not None:

View file

@ -42,6 +42,7 @@ class BaseTokenCounter(ABC):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
pass

View file

@ -13,7 +13,13 @@ from typing_extensions import assert_never
import litellm
from litellm._logging import verbose_logger
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields
from litellm.llms.anthropic.count_tokens.transformation import (
NO_CLAUDE_COUNT_OPTIONS,
ChatCountTranslationError,
ClaudeCountOptions,
anthropic_count_fields,
claude_count_options,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
@ -74,6 +80,13 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
)
def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]:
return {
**claude_options.options,
**({"anthropic_beta": list(claude_options.betas)} if claude_options.betas else {}),
}
def _request_data(
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, system: object
) -> Mapping[str, object]:
@ -143,6 +156,7 @@ class BedrockTokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
if not messages:
return None
@ -167,11 +181,27 @@ class BedrockTokenCounter(BaseTokenCounter):
)
if isinstance(translated, ChatCountTranslationError):
return translated.response(request_model, model_to_use, primary_tokenizer)
runtime_request: Final = (
_request_data(model_to_use, messages, tools, system)
if translated is None
else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
claude_options: Final = (
claude_count_options(
counted_model,
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
request_options,
context_management,
deployment,
)
if is_mantle_claude_model(resolved_model)
else NO_CLAUDE_COUNT_OPTIONS
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, primary_tokenizer)
runtime_request: Final = {
**(
_request_data(model_to_use, messages, tools, system)
if translated is None
else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
),
**_invoke_fields(claude_options),
}
mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
primary: Final = (

View file

@ -26,6 +26,7 @@ class MantleCountTokensRequest(TypedDict):
_COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
_BETAS: Final = TypeAdapter(tuple[str, ...])
class BedrockMantleCountTokensHandler(BaseAWSLLM):
@ -70,10 +71,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
)
verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
api_key: Final = litellm_params.get("api_key")
betas: Final = _BETAS.validate_python(request_data.get("anthropic_beta", ()))
signed_headers, signed_body = await run_aws_signing(
self._sign_request,
service_name="bedrock",
headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
headers={
"Content-Type": "application/json",
"anthropic-version": MANTLE_ANTHROPIC_VERSION,
**({"anthropic-beta": ",".join(betas)} if betas else {}),
},
optional_params=litellm_params,
request_data=body,
api_base=endpoint_url,

View file

@ -19,6 +19,16 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
# max_tokens field; CountTokens only counts input, so it has no effect
# on any generation.
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024
_INVOKE_MODEL_ONLY_FIELDS: Final = frozenset(
("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta")
)
def _invoke_model_max_tokens(thinking: JsonValue) -> int:
budget: Final = thinking.get("budget_tokens") if isinstance(thinking, dict) else None
if not isinstance(budget, int):
return DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
return max(DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, budget + 1)
def _json_dict(value: JsonValue) -> dict[str, JsonValue]:
@ -87,6 +97,8 @@ class BedrockCountTokensConfig(BaseAWSLLM):
'converse' or 'invokeModel'
"""
messages: Final = request_data.get("messages")
if not _INVOKE_MODEL_ONLY_FIELDS.isdisjoint(request_data):
return "invokeModel"
if isinstance(messages, list):
# Anthropic content blocks carry a "type" key ({"type": "text", ...});
# Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
@ -200,7 +212,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
# Bedrock validates the body against the model's InvokeModel schema;
# Anthropic Messages bodies require these fields.
body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
body_data.setdefault("max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS)
body_data.setdefault("max_tokens", _invoke_model_max_tokens(body_data.get("thinking")))
# The CountTokens API expects invokeModel.body as a base64-encoded blob
encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode()

View file

@ -534,6 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
import copy

View file

@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
"""
Count tokens using OpenAI's Responses API /input_tokens endpoint.

View file

@ -30,7 +30,7 @@ from litellm.types.llms.vertex_ai import (
PartType,
Schema,
)
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.utils import supports_response_schema, supports_system_messages
GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
@ -1391,8 +1391,6 @@ class VertexAITokenCounter(BaseTokenCounter):
self,
custom_llm_provider: str | None = None,
) -> bool:
from litellm.types.utils import LlmProviders
return custom_llm_provider == LlmProviders.VERTEX_AI.value
async def _count_translated_claude_request(
@ -1454,6 +1452,7 @@ class VertexAITokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
import copy
@ -1508,6 +1507,17 @@ class VertexAITokenCounter(BaseTokenCounter):
if translated_count is not None:
return translated_count
from litellm.llms.anthropic.count_tokens.transformation import (
ChatCountTranslationError,
claude_count_options,
)
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.VERTEX_AI, request_options, context_management, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
# Use partner models token counter
partner_models_handler: Final = VertexAIPartnerModels()
@ -1538,7 +1548,8 @@ class VertexAITokenCounter(BaseTokenCounter):
vertex_credentials=vertex_credentials,
system=system,
tools=tools,
count_options=count_options,
count_options={**count_options, **claude_options.options},
betas=claude_options.betas,
)
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
status_code: Final = e.response.status_code

View file

@ -3,7 +3,7 @@ from typing import Final
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
# Check for web search tool
for tool in tools:

View file

@ -6,6 +6,7 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Sequence
from typing import Any, Final
from litellm.litellm_core_utils.url_utils import join_api_base
@ -90,6 +91,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
model: str,
request_data: dict[str, Any],
litellm_params: dict[str, Any],
betas: Sequence[str] = (),
) -> dict[str, Any]:
"""
Handle token counting request for a Vertex AI partner model.
@ -98,6 +100,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
model: The model name
request_data: Request payload (Anthropic Messages API format)
litellm_params: LiteLLM parameters containing credentials, project, location
betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header
Returns:
Dict containing token count information
@ -157,7 +160,10 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
)
# Prepare headers
headers: Final = {"Authorization": f"Bearer {access_token}"}
headers: Final = {
"Authorization": f"Bearer {access_token}",
**({"anthropic-beta": ",".join(betas)} if betas else {}),
}
# Get async HTTP client
from litellm import LlmProviders

View file

@ -1,6 +1,6 @@
# What is this?
## API Handler for calling Vertex AI Partner Models
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Final
@ -267,6 +267,7 @@ class VertexAIPartnerModels(VertexBase):
system: object | None = None,
tools: list[dict[str, object]] | None = None,
count_options: Mapping[str, object] = MappingProxyType({}),
betas: Sequence[str] = (),
):
"""
Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
@ -321,6 +322,7 @@ class VertexAIPartnerModels(VertexBase):
model=model,
request_data=request_data,
litellm_params=_litellm_params,
betas=betas,
)
return result

View file

@ -336,12 +336,14 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format")
class _CountTokensBodyOptions(TypedDict):
output_format: ReadOnly[JsonValue]
context_management: ReadOnly[JsonValue]
request_litellm_params: ReadOnly[Mapping[str, JsonValue]]
request_options: ReadOnly[Mapping[str, JsonValue]]
def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions:
@ -352,6 +354,9 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
"request_litellm_params": MappingProxyType(
{"drop_params": True, **sent_params} if auto_drop_params else sent_params
),
"request_options": MappingProxyType(
{key: data[key] for key in _COUNTED_REQUEST_OPTIONS if data.get(key) is not None}
),
}

View file

@ -299,7 +299,7 @@ def generate_feedback_box():
import contextlib
from collections import defaultdict
from contextlib import AsyncExitStack, asynccontextmanager
from functools import lru_cache, partial
from functools import lru_cache, partial, reduce
import litellm
import litellm._redis
@ -1374,6 +1374,8 @@ _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody)
_LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody)
_ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig)
_LITELLM_PARAMS_COPY: Final = TypeAdapter(dict[str, object])
_COUNT_REQUEST_OPTIONS: Final = TypeAdapter(Mapping[str, JsonValue])
_DROPPED_PARAM_PATHS: Final = TypeAdapter(tuple[object, ...])
_WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings)
_UNTYPED_VALUE: Final = TypeAdapter(object)
@ -14810,6 +14812,7 @@ async def _try_provider_token_count(
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
@ -14837,6 +14840,7 @@ async def _try_provider_token_count(
output_format=output_format,
context_management=context_management,
chat_params=chat_params,
request_options=request_options,
)
except httpx.HTTPStatusError as e:
return ProviderCountFailure(
@ -15239,6 +15243,20 @@ def _count_context_management(
)
def _count_request_options(
request_options: Mapping[str, JsonValue],
deployment: Mapping[str, object] | None,
sent_params: Mapping[str, object],
) -> Mapping[str, JsonValue]:
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
deployment_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
dropped: Final = {**deployment_params, **sent_params}.get("additional_drop_params")
dropped_values: Final = _DROPPED_PARAM_PATHS.validate_python(dropped) if isinstance(dropped, list) else ()
dropped_paths: Final = tuple(path for path in dropped_values if isinstance(path, str))
return _COUNT_REQUEST_OPTIONS.validate_python(reduce(delete_nested_value, dropped_paths, dict(request_options)))
def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> Mapping[str, object]:
previous_response_id: Final = None if responses_request is None else responses_request.get("previous_response_id")
return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {}
@ -15316,6 +15334,7 @@ async def count_request_tokens(
responses_request: Mapping[str, object] | None = None,
litellm_call_id: str | None = None,
client_oauth_token: str | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse:
call_id: Final = resolve_litellm_call_id(litellm_call_id)
if request.prompt is None and request.messages is None and request.contents is None:
@ -15335,6 +15354,7 @@ async def count_request_tokens(
request_litellm_params=request_litellm_params,
responses_request=responses_request,
client_oauth_token=client_oauth_token,
request_options=request_options,
)
return await logged_token_count(
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
@ -15377,6 +15397,7 @@ async def _count_request_tokens(
request_litellm_params: Mapping[str, JsonValue],
responses_request: Mapping[str, object] | None,
client_oauth_token: str | None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse:
prompt: Final = request.prompt
messages: Final = request.messages
@ -15484,6 +15505,7 @@ async def _count_request_tokens(
output_format=output_format,
context_management=_count_context_management(context_management, request.model, request_litellm_params),
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
request_options=_count_request_options(request_options, count_deployment, sent_params),
),
model_to_use=model_to_use,
request_model=request.model,

View file

@ -13,6 +13,7 @@ import respx
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
@ -26,7 +27,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
@pytest.mark.parametrize(
@ -475,6 +477,43 @@ _COUNTERS: Final = pytest.mark.parametrize(
)
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
async def _counted_requests(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
messages: list[dict[str, JsonValue]],
tools: list[dict[str, JsonValue]] | None,
messages_format: TokenCountMessagesFormat,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
context_management: CountContextManagement | None = None,
model: str = "claude-opus-5-5",
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def count(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(200, json={"input_tokens": 31})
with respx.mock:
respx.post(count_endpoint).mock(side_effect=count)
result: Final = await counter_type().count_tokens(
model_to_use=model,
messages=messages,
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
request_model=model,
tools=tools,
messages_format=messages_format,
chat_params=chat_params,
request_options=request_options,
context_management=context_management,
)
return result, tuple(sent.get() for _ in range(sent.qsize()))
async def _counted_body(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
@ -483,25 +522,10 @@ async def _counted_body(
messages_format: TokenCountMessagesFormat,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]:
sent: Final[SimpleQueue[bytes]] = SimpleQueue()
def count(request: httpx.Request) -> httpx.Response:
sent.put(request.content)
return httpx.Response(200, json={"input_tokens": 31})
with respx.mock:
respx.post(count_endpoint).mock(side_effect=count)
result: Final = await counter_type().count_tokens(
model_to_use="claude-opus-5-5",
messages=messages,
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
request_model="claude-opus-5-5",
tools=tools,
messages_format=messages_format,
chat_params=chat_params,
)
return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize()))
result, requests = await _counted_requests(
counter_type, count_endpoint, messages, tools, messages_format, chat_params
)
return result, tuple(_JSON_BODY.validate_json(request.content) for request in requests)
@pytest.mark.asyncio
@ -842,3 +866,134 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat
inference["tool_choice"],
inference["tools"],
)
_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
_INFERENCE_PROVIDERS: Final = MappingProxyType(
{AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"}
)
_CONTEXT_BETAS: Final = frozenset(
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
)
)
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}
_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000}
async def _messages_inference_request(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
inference_endpoint: str,
request_options: Mapping[str, JsonValue],
) -> httpx.Request:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def reply(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(200, json=_MESSAGE_REPLY)
with respx.mock:
respx.post(inference_endpoint).mock(side_effect=reply)
await litellm.anthropic_messages(
model=f"{_INFERENCE_PROVIDERS[counter_type]}/claude-haiku-5-5",
messages=[{"role": "user", "content": "Weather in SF?"}],
tools=[_WEATHER_TOOL],
max_tokens=4096,
api_key="sk-ant-api03-test-key",
api_base="https://gateway.example",
**request_options,
)
return sent.get()
def _betas(request: httpx.Request) -> frozenset[str]:
return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
@pytest.mark.parametrize(
("request_options", "context_management"),
(
({"thinking": _ENABLED_THINKING}, None),
({"tool_choice": {"type": "any"}}, None),
({"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, None),
(
{"thinking": _ENABLED_THINKING},
{"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]},
),
({}, {"edits": [{"type": "compact_20260112"}]}),
),
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
)
async def test_counter_sends_the_options_and_context_edits_messages_inference_sends(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
request_options: dict[str, JsonValue],
context_management: dict[str, JsonValue] | None,
) -> None:
"""A /v1/messages count that drops thinking, tool_choice, output_config or context_management
counts a different prompt than inference bills, so it carries them the way inference sends them."""
inference: Final = await _messages_inference_request(
counter_type,
count_endpoint.removesuffix("/count_tokens"),
{**request_options, **({} if context_management is None else {"context_management": context_management})},
)
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
request_options=request_options,
context_management=(
None if context_management is None else CountContextManagement(spec=context_management, drop_invalid_edits=False)
),
model="claude-haiku-5-5",
)
inference_body: Final = _JSON_BODY.validate_json(inference.content)
count_body: Final = _JSON_BODY.validate_json(count.content)
inference_output_format: Final = inference_body.get("output_format")
inference_output_config: Final = inference_body.get("output_config")
assert "output_format" not in count_body
assert count_body.get("output_config") == (
inference_output_config
if inference_output_format is None
else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format}
)
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
}
assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_refuses_options_messages_inference_refuses_without_calling_anthropic(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
result, requests = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
None,
"anthropic",
request_options={"output_config": {"effort": "fastest"}},
model="claude-haiku-4-5",
)
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
assert requests == ()

View file

@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import CountContextManagement
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
@ -338,6 +339,105 @@ async def test_claude_count_sends_the_tool_search_tool_the_deployments_inference
assert counted_tools == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
}
_THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
"thinking": {"type": "enabled", "budget_tokens": 2000},
"tool_choice": {"type": "auto"},
}
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
async def _messages_inference_and_count(deployment_model: str, model_to_use: str) -> tuple[_Bedrock, _Bedrock]:
credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages(
model=deployment_model,
messages=MESSAGES,
tools=[_WEATHER_TOOL],
max_tokens=4096,
context_management=_CONTEXT_EDITS,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**_THINKING_AND_TOOL_CHOICE,
**credentials,
)
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use=model_to_use,
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}},
request_model="claude",
tools=[_WEATHER_TOOL],
request_options=_THINKING_AND_TOOL_CHOICE,
context_management=CountContextManagement(spec=_CONTEXT_EDITS, drop_invalid_edits=False),
)
assert result is not None
assert result.error is False
return inference, bedrock
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "model_to_use"),
(
("bedrock/invoke/global.anthropic.claude-haiku-5-5", "global.anthropic.claude-haiku-5-5"),
("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"),
),
)
async def test_runtime_count_sends_the_options_and_context_edits_invoke_inference_sends(
deployment_model: str, model_to_use: str
) -> None:
"""Bedrock invoke inference drops edits InvokeModel rejects and sends its betas in the body, so a count
that skips either counts a prompt inference never sends, or gets the 400 inference avoids."""
inference, bedrock = await _messages_inference_and_count(deployment_model, model_to_use)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _invoke_body(bedrock.requests[0])
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
thinking: Final = counted["thinking"]
assert isinstance(thinking, dict)
budget_tokens: Final = thinking.get("budget_tokens", 0)
assert isinstance(budget_tokens, int) and isinstance(counted["max_tokens"], int)
assert counted["max_tokens"] > budget_tokens
@pytest.mark.asyncio
async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5"
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
assert "anthropic_beta" not in counted
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
@pytest.mark.asyncio
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-sonnet-4-5",
request_options={"output_config": {"effort": "fastest"}},
)
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
assert bedrock.posted_hosts() == ()
@pytest.mark.asyncio
async def test_claude_chat_tool_use_counted_on_mantle_metadata_route_is_translated_too() -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))

View file

@ -1,8 +1,10 @@
import copy
import time
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from enum import Enum
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, patch
@ -1840,7 +1842,9 @@ _PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = {
async def _partner_counted_bodies(
model: str, chat_params: dict[str, JsonValue] | None = None
model: str,
chat_params: dict[str, JsonValue] | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> tuple[dict[str, object], ...]:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
@ -1875,6 +1879,7 @@ async def _partner_counted_bodies(
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=[_PARTNER_RESPONSES_TOOL],
chat_params=chat_params or {},
request_options=request_options,
)
assert result is not None
@ -1954,6 +1959,101 @@ async def test_vertex_claude_count_sends_the_structured_output_vertex_inference_
)
_PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
@pytest.mark.asyncio
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
context_edits: Final[dict[str, JsonValue]] = {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
}
options: Final[dict[str, JsonValue]] = {
"thinking": {"type": "enabled", "budget_tokens": 2000},
"tool_choice": {"type": "auto"},
}
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
tools: Final[list[dict[str, JsonValue]]] = [{"name": "get_weather", "input_schema": {"type": "object"}}]
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def vertex(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(
200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY
)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex))
with (
patch.object(handler, "get_async_httpx_client", return_value=client),
patch.object(
VertexBase, "_ensure_access_token_async", new=AsyncMock(return_value=("fake-token", "test-project"))
),
):
await litellm.anthropic_messages(
model="vertex_ai/claude-haiku-5-5",
messages=messages,
tools=tools,
max_tokens=4096,
context_management=context_edits,
vertex_project="test-project",
vertex_location="us-east5",
client=client,
**options,
)
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use="claude-haiku-5-5",
messages=messages,
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=tools,
request_options=options,
context_management=CountContextManagement(spec=context_edits, drop_invalid_edits=False),
)
assert result is not None
assert result.total_tokens == 37
inference, count = tuple(sent.get() for _ in range(sent.qsize()))
assert count.url.path.endswith("count-tokens:rawPredict")
counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content)
inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
assert {name: counted.get(name) for name in fields} == {name: inferred.get(name) for name in fields}
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
@pytest.mark.asyncio
async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None:
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use="claude-haiku-4-5",
messages=[{"role": "user", "content": "Weather in SF?"}],
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
request_options={"output_config": {"effort": "fastest"}},
)
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
@pytest.mark.asyncio
async def test_vertex_llama_count_leaves_out_the_claude_only_options() -> None:
model: Final = "meta/llama-4-maverick-17b-128e-instruct-maas"
assert await _partner_counted_bodies(
model, request_options={"thinking": {"type": "enabled", "budget_tokens": 2000}}
) == await _partner_counted_bodies(model)
@pytest.mark.asyncio
async def test_vertex_llama_count_sends_chat_tool_use_as_sent() -> None:
assert await _partner_counted_bodies("meta/llama-4-maverick-17b-128e-instruct-maas") == (

View file

@ -367,6 +367,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
request_options=None,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "claude-3-sonnet-20240229"
@ -449,6 +450,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
request_options=None,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "gpt-4"
@ -1269,6 +1271,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
request_options=None,
):
raise ProxyException(
message='{"detail":{"message":"Input is too long for requested model."}}',
@ -1328,6 +1331,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
request_options=None,
):
raise ProxyException(
message='{"Message":"Bearer Token has expired"}',
@ -1387,6 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
request_options=None,
):
raise ProxyException(
message="Rate limit exceeded",
@ -1607,6 +1612,103 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t
assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}]
_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = {
"thinking": {"type": "adaptive"},
"tool_choice": {"type": "auto"},
"output_config": {
"effort": "low",
"format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
},
"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
}
def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
output_config: Final = body.get("output_config")
return frozenset(name for name in _CLAUDE_COUNT_OPTIONS if name in body) | frozenset(
f"output_config.{key}" for key in (output_config if isinstance(output_config, dict) else {})
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("additional_drop_params", "expected_paths"),
(
(
None,
{
"thinking",
"tool_choice",
"output_config",
"output_config.effort",
"output_config.format",
"context_management",
},
),
(
["tool_choice", "output_config.format"],
{"thinking", "output_config", "output_config.effort", "context_management"},
),
(
["context_management"],
{"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"},
),
),
ids=("nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management"),
)
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
monkeypatch: pytest.MonkeyPatch, additional_drop_params: list[str] | None, expected_paths: set[str]
) -> None:
import litellm.proxy.proxy_server as proxy_server
router: Final = Router(
model_list=[
{
"model_name": "claude-agent",
"litellm_params": {
"model": "anthropic/claude-haiku-5-5",
"api_key": "sk-ant-test",
"additional_drop_params": additional_drop_params,
},
}
]
)
request: Final = _json_request(
{
"model": "claude-agent",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
**_CLAUDE_COUNT_OPTIONS,
},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 7})
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
assert response["input_tokens"] == 7
sent_request: Final = route.calls.last.request
assert _sent_option_paths(json.loads(sent_request.content)) == expected_paths
assert ("context-management-2025-06-27" in sent_request.headers["anthropic-beta"]) == (
"context_management" in expected_paths
)
class _ScriptedCounter(BaseTokenCounter):
def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None:
self._replies = replies
@ -1631,6 +1733,7 @@ class _ScriptedCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: object = None,
chat_params: object = None,
request_options: object = None,
) -> TokenCountResponse | None:
self.calls.append((model_to_use, messages, tools, system))
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
@ -3470,7 +3573,7 @@ class _PartnerCountRequest(BaseModel):
async def _vertex_partner_word_count(
model: str, request_data: JsonValue, litellm_params: JsonValue
model: str, request_data: JsonValue, litellm_params: JsonValue, betas: object = ()
) -> dict[str, JsonValue]:
body: Final = _PartnerCountRequest.model_validate(request_data)
return {"input_tokens": sum(len(message.content.split()) for message in body.messages)}