fix(count_tokens): forward context_management, thinking, tool_choice and output_config to the Claude count APIs

Backport of the count_tokens context_management fix (f012c60033, 875a0e6404, a6f4acdb6f, 408372dbcb,
8db6f0d5ed, f84c244f3e, 14b572d44b, 502ce4a887 and 0fa2f90ab6 on litellm_count_tokens_forward_context_management)
to lts/1.108.x, adapted to the line's count path. The count sends inference's betas and tools when it
forwards options, counts with the deployment's model, adds the effort beta on Bedrock runtime counts,
strips disable_parallel_tool_use on Converse the way Converse inference does, lets the caller's
drop_params replace the deployment's like inference (an explicit null included), and counts once more
without the forwarded options when the provider rejects them with a 4xx other than 401, 403 or 429
This commit is contained in:
mateo-berri 2026-11-06 13:31:59 -08:00
parent e03de97a98
commit 30bfb30dd0
30 changed files with 2452 additions and 186 deletions

View file

@ -12,7 +12,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -43,6 +43,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -345,6 +346,18 @@ def _beta_header_values(side: str | Sequence[str] | None) -> tuple[str, ...]:
return tuple(entry for entry in side if isinstance(entry, str))
def context_management_betas(context_management: JsonValue) -> frozenset[str]:
edits: Final = context_management.get("edits") if isinstance(context_management, dict) else None
if not isinstance(edits, (list, tuple)):
return frozenset()
return frozenset(
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
if isinstance(edit, dict) and edit.get("type") == "compact_20260112"
else ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
for edit in edits
)
def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta: str | Sequence[str] | None) -> str:
"""Merge anthropic-beta header values, deduplicated and sorted.
@ -1283,8 +1296,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

View file

@ -4,7 +4,7 @@ Anthropic CountTokens API handler.
Uses httpx for HTTP requests instead of the Anthropic SDK.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
betas: Sequence[str] = (),
) -> dict[str, JsonValue]:
"""
Handle a CountTokens request using httpx.
@ -79,7 +80,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, betas)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

View file

@ -2,6 +2,8 @@
Anthropic Token Counter implementation using the CountTokens API.
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final
from pydantic import JsonValue
@ -10,6 +12,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, claude_count_options
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
@ -38,6 +41,8 @@ class AnthropicTokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Anthropic's CountTokens API.
@ -57,6 +62,12 @@ class AnthropicTokenCounter(BaseTokenCounter):
if not messages:
return None
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, anthropic_beta, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "anthropic_api")
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
api_base: Final = litellm_params.get("api_base")
@ -84,11 +95,17 @@ class AnthropicTokenCounter(BaseTokenCounter):
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,
messages=messages,
messages=messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
auth_header=auth_header,
api_base=api_base,
tools=tools,
tools=tools
if claude_options.messages is None
else [dict(tool) for tool in claude_options.tools or ()] or None,
system=system,
optional_params=claude_options.options,
betas=claude_options.betas,
)
if result is not None:

View file

@ -4,11 +4,14 @@ Anthropic CountTokens API transformation logic.
This module handles the transformation of requests to Anthropic's CountTokens API format.
"""
import copy
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from typing import TYPE_CHECKING, Final
from pydantic import JsonValue, TypeAdapter
from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
@ -17,12 +20,17 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
from litellm.types.llms.openai import ChatCompletionImageObject
from litellm.types.utils import LlmProviders, TokenCountResponse
if TYPE_CHECKING:
from litellm.types.router import GenericLiteLLMParams
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config", "context_management")
def _count_image(block: JsonValue) -> JsonValue:
@ -74,6 +82,202 @@ def _lift_leading_system(
return conversation, lifted or system
@dataclass(frozen=True, slots=True)
class ChatCountTranslationError:
message: str
def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse:
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_used,
tokenizer_type=tokenizer_type,
error=True,
error_message=self.message,
status_code=400,
rejected_before_provider_call=True,
)
@with_config(ConfigDict(extra="ignore"))
class _CountedOptions(TypedDict):
thinking: NotRequired[ReadOnly[JsonValue]]
tool_choice: NotRequired[ReadOnly[JsonValue]]
output_config: NotRequired[ReadOnly[JsonValue]]
output_format: NotRequired[ReadOnly[JsonValue]]
context_management: NotRequired[ReadOnly[JsonValue]]
@with_config(ConfigDict(extra="ignore"))
class _AnthropicChatRequest(_CountedOptions):
messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]]
tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]]
_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest)
_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object])
_HEADERS: Final = TypeAdapter(dict[str, str])
def _count_options(request: _CountedOptions) -> Mapping[str, JsonValue]:
output_config: Final = request.get("output_config")
output_format: Final = request.get("output_format")
counted_config: Final = {
**(output_config if isinstance(output_config, dict) else {}),
**({} if output_format is None else {"format": output_format}),
}
return MappingProxyType(
{
key: value
for key, value in (
("thinking", request.get("thinking")),
("tool_choice", request.get("tool_choice")),
("output_config", counted_config or None),
("context_management", request.get("context_management")),
)
if value is not None
}
)
@dataclass(frozen=True, slots=True)
class ClaudeCountOptions:
options: Mapping[str, JsonValue]
betas: tuple[str, ...]
messages: tuple[Mapping[str, JsonValue], ...] | None
tools: tuple[Mapping[str, JsonValue], ...] | None
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
_STATUSES_NOT_CAUSED_BY_THE_REQUEST: Final = frozenset({401, 403, 429})
def is_request_rejection(status_code: int) -> bool:
return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST
def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None:
deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {})
deployment_model: Final = deployment_params.get("model")
if not isinstance(deployment_model, str):
return None
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
provider_hint: Final = deployment_params.get("custom_llm_provider")
api_base: Final = deployment_params.get("api_base")
inference_model, _, _, _ = get_llm_provider(
model=deployment_model,
custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)
return inference_model
def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None:
from litellm.utils import ProviderConfigManager
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
return None if messages_config is None else (model, messages_config)
def _transform_litellm_params(deployment: Mapping[str, object] | None) -> "GenericLiteLLMParams":
from litellm.types.router import GenericLiteLLMParams
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
return GenericLiteLLMParams.model_validate(
{
"drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"),
}
)
def _claude_count_options_for(
messages_config: BaseAnthropicMessagesConfig,
model: str,
litellm_params: "GenericLiteLLMParams",
provider: LlmProviders,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
sent: Mapping[str, JsonValue],
anthropic_beta: str | None,
) -> ClaudeCountOptions:
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_params: Final = {
**copy.deepcopy(sent),
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
}
beta_headers: Final = messages_config.headers_with_anthropic_betas(
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
)
sent_headers: Final = (
_HEADERS.validate_python(
update_headers_with_filtered_beta(
headers=dict(beta_headers),
provider=provider.value,
keep=messages_config.model_specific_anthropic_betas(model, litellm_params),
)
)
if messages_config.should_filter_anthropic_beta_headers()
else dict(beta_headers)
)
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
messages=sent_conversation,
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
litellm_params=litellm_params,
headers=sent_headers,
)
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
return ClaudeCountOptions(
options=_count_options(transformed),
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
tools=transformed.get("tools"),
)
def claude_count_options(
model: str,
provider: LlmProviders,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
request_options: Mapping[str, JsonValue],
anthropic_beta: str | None,
deployment: Mapping[str, object] | None,
) -> ClaudeCountOptions | ChatCountTranslationError:
sent: Final = {name: value for name, value in request_options.items() if value is not None}
if not sent:
return NO_CLAUDE_COUNT_OPTIONS
inference_model: Final = deployment_inference_model(deployment)
transform: Final = (
None if inference_model is None else _messages_transform(inference_model, provider)
) or _messages_transform(model, provider)
if transform is None:
return NO_CLAUDE_COUNT_OPTIONS
transform_model, messages_config = transform
litellm_params: Final = _transform_litellm_params(deployment)
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError
try:
return _claude_count_options_for(
messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
except (AttributeError, KeyError, TypeError, ValueError) as e:
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
class AnthropicCountTokensConfig:
"""
Configuration and transformation logic for Anthropic CountTokens API.
@ -137,7 +341,7 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(self, auth_header: Mapping[str, str], betas: Sequence[str] = ()) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
@ -147,7 +351,7 @@ class AnthropicCountTokensConfig:
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
auth_header.get("anthropic-beta"), (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *betas)
),
}

View file

@ -26,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
AnthropicError,
AnthropicModelInfo,
context_management_betas,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
@ -378,6 +379,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
messages=messages,
)
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return self._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared beta step takes and returns bare dicts
headers=dict(headers),
optional_params=dict(optional_params),
messages=messages,
)
@staticmethod
def _translate_reasoning_effort_to_anthropic(
model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str
@ -742,28 +755,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

View file

@ -4,6 +4,7 @@ Azure AI Anthropic CountTokens API handler.
Uses httpx for HTTP requests with Azure authentication.
"""
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
@ -12,7 +13,7 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.common_utils import AnthropicError, merge_anthropic_beta_headers
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
@ -36,6 +37,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
timeout: float | httpx.Timeout | None = None,
tools: list[dict[str, JsonValue]] | None = None,
system: object = None,
optional_params: Mapping[str, JsonValue] | None = None,
betas: Sequence[str] = (),
) -> dict[str, object]:
"""
Handle a CountTokens request using httpx with Azure authentication.
@ -66,6 +69,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
messages=messages,
tools=tools,
system=system,
optional_params=optional_params,
)
verbose_logger.debug("Transformed request: %s", request_body)
@ -98,7 +102,10 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
response: Final = await async_client.post(
endpoint_url,
headers=headers,
headers={
**headers,
"anthropic-beta": merge_anthropic_beta_headers(headers.get("anthropic-beta"), betas),
},
json=request_body,
timeout=request_timeout,
)

View file

@ -3,12 +3,15 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API.
"""
import os
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final
from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, claude_count_options
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
@ -55,6 +58,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Azure AI Anthropic's CountTokens API.
@ -74,6 +79,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
if not messages:
return None
claude_options: Final = claude_count_options(
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, anthropic_beta, deployment
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
@ -95,12 +106,18 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
try:
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,
messages=messages,
messages=messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
api_key=api_key,
api_base=api_base,
litellm_params=litellm_params,
tools=tools,
tools=tools
if claude_options.messages is None
else [dict(tool) for tool in claude_options.tools or ()] or None,
system=system,
optional_params=claude_options.options,
betas=claude_options.betas,
)
if result is not None:

View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -127,6 +127,18 @@ class BaseAnthropicMessagesConfig(ABC):
"""
return headers, None
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return headers
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
beta_header: Final = next((value for name, value in headers.items() if name.lower() == "anthropic-beta"), "")
return tuple(sorted(frozenset(beta.strip() for beta in beta_header.split(",")) - {""}))
def should_filter_anthropic_beta_headers(self) -> bool:
"""
Whether ``anthropic-beta`` header values should be filtered down to the

View file

@ -8,6 +8,7 @@ import json
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final
from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals
@ -39,6 +40,8 @@ class BaseTokenCounter(ABC):
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
pass

View file

@ -2,20 +2,30 @@
Bedrock Token Counter implementation using the CountTokens API.
"""
from collections.abc import Mapping, Sequence
from collections.abc import Awaitable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final
from pydantic import JsonValue
from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm.llms.anthropic.count_tokens.transformation import (
NO_CLAUDE_COUNT_OPTIONS,
ChatCountTranslationError,
ClaudeCountOptions,
claude_count_options,
deployment_inference_model,
is_request_rejection,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER
from litellm.types.utils import (
CountContextManagement,
LiteLLMPydanticObjectBase,
@ -58,6 +68,48 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
)
def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options: bool) -> bool:
return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code)
def _invoke_fields(options: Mapping[str, JsonValue], betas: Sequence[str]) -> Mapping[str, object]:
return {**options, **({"anthropic_beta": list(betas)} if betas else {})}
def _sets_effort(options: Mapping[str, JsonValue]) -> bool:
output_config: Final = options.get("output_config")
return isinstance(output_config, dict) and "effort" in output_config
def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]:
if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas:
return claude_options.betas
return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER)
def _runs_on_converse(inference_model: str) -> bool:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(inference_model) is None
def _options_converse_sends(
request_options: Mapping[str, JsonValue], inference_model: str | None
) -> Mapping[str, JsonValue]:
tool_choice: Final = request_options.get("tool_choice")
if (
not isinstance(tool_choice, dict)
or "disable_parallel_tool_use" not in tool_choice
or inference_model is None
or not _runs_on_converse(inference_model)
):
return request_options
return {
**request_options,
"tool_choice": {key: value for key, value in tool_choice.items() if key != "disable_parallel_tool_use"},
}
class BedrockTokenCounter(BaseTokenCounter):
"""Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model"""
@ -77,21 +129,19 @@ class BedrockTokenCounter(BaseTokenCounter):
) -> bool:
return custom_llm_provider == LlmProviders.BEDROCK.value
async def _count_with(
self,
handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler,
tokenizer_type: str,
request_data: dict[str, object],
litellm_params: dict[str, object],
resolved_model: str,
) -> CountTokensOutcome:
def _count_on_mantle(
self, request_data: Mapping[str, object], litellm_params: Mapping[str, object], resolved_model: str
) -> Awaitable[Mapping[str, JsonValue]]:
return self._mantle_handler.handle_count_tokens_request(
request_data=dict(request_data),
litellm_params=dict(litellm_params),
resolved_model=resolved_model,
client=self._client,
)
async def _count_with(self, count: Awaitable[Mapping[str, JsonValue]], tokenizer_type: str) -> CountTokensOutcome:
try:
result: Final = await handler.handle_count_tokens_request(
request_data=request_data,
litellm_params=litellm_params,
resolved_model=resolved_model,
client=self._client,
)
result: Final = await count
reply: Final = _CountTokensReply.model_validate(result)
except BedrockError as e:
verbose_logger.debug(
@ -115,32 +165,74 @@ class BedrockTokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
if not messages:
return None
use_mantle: Final = model_to_use.startswith("bedrock_mantle/")
counted_model: Final = model_to_use.removeprefix("bedrock_mantle/")
resolved_model: Final = get_bedrock_base_model(counted_model)
primary_tokenizer: Final = MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE
litellm_params: Final = (deployment or {}).get("litellm_params", {})
request_data: Final[dict[str, object]] = {
claude_options: Final = (
claude_count_options(
counted_model,
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
messages,
tools,
_options_converse_sends(request_options, deployment_inference_model(deployment)),
anthropic_beta,
deployment,
)
if is_mantle_claude_model(resolved_model)
else NO_CLAUDE_COUNT_OPTIONS
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, primary_tokenizer)
invoke_fields: Final = _invoke_fields(claude_options.options, claude_options.betas)
sent_tools: Final = tools if claude_options.messages is None else claude_options.tools
counted_request: Final[Mapping[str, object]] = {
"model": model_to_use,
"messages": messages,
**({"tools": tools} if tools else {}),
"messages": messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
**({"tools": [dict(tool) for tool in sent_tools]} if sent_tools else {}),
**({"system": system} if system else {}),
}
use_mantle: Final = model_to_use.startswith("bedrock_mantle/")
resolved_model: Final = get_bedrock_base_model(model_to_use.removeprefix("bedrock_mantle/"))
mantle_request: Final = {**counted_request, **invoke_fields}
runtime_request: Final = {
**counted_request,
**_invoke_fields(claude_options.options, _runtime_count_betas(claude_options)),
}
primary: Final = await self._count_with(
self._mantle_handler if use_mantle else self._runtime_handler,
MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE,
request_data,
litellm_params,
resolved_model,
)
outcome: Final = (
primary: Final = (
await self._count_with(
self._mantle_handler, MANTLE_TOKENIZER_TYPE, request_data, litellm_params, resolved_model
self._count_on_mantle(mantle_request, litellm_params, resolved_model), primary_tokenizer
)
if use_mantle
else await self._count_with(
self._runtime_handler.handle_count_tokens_request(
request_data=runtime_request,
litellm_params=litellm_params,
resolved_model=resolved_model,
client=self._client,
input_type="invokeModel" if invoke_fields else None,
),
primary_tokenizer,
)
)
fallback: Final = (
await self._count_with(
self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE
)
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
else None
)
outcome: Final = (
fallback
if fallback is not None and _mantle_answered_for_runtime(fallback, forwarded_options=bool(invoke_fields))
else primary
)
match outcome:

View file

@ -12,7 +12,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.base_aws_llm import run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig, BedrockCountTokensInputType
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
@ -29,6 +29,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
litellm_params: dict[str, Any],
resolved_model: str,
client: AsyncHTTPHandler | None = None,
input_type: BedrockCountTokensInputType | None = None,
) -> dict[str, Any]:
"""
Handle a CountTokens request using existing LiteLLM patterns.
@ -43,7 +44,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
"""
try:
# Validate the request
self.validate_count_tokens_request(request_data)
self.validate_count_tokens_request(request_data, input_type)
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
@ -57,7 +58,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug("Retrieved AWS region: %s", aws_region_name)
# Transform request to Bedrock format (supports both Converse and InvokeModel)
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data)
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(
request_data=request_data, input_type=input_type
)
verbose_logger.debug("Transformed request: %s", bedrock_request)

View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Final
import httpx
@ -6,7 +7,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountTokensConfig
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
@ -23,6 +24,8 @@ class MantleCountTokensRequest(TypedDict):
_COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
_BETAS: Final = TypeAdapter(tuple[str, ...])
class BedrockMantleCountTokensHandler(BaseAWSLLM):
@ -61,13 +64,21 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
messages=request["messages"],
tools=request.get("tools"),
system=request.get("system"),
optional_params=_COUNT_OPTIONS.validate_python(
{name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}
),
)
verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
api_key: Final = litellm_params.get("api_key")
betas: Final = _BETAS.validate_python(request_data.get("anthropic_beta", ()))
signed_headers, signed_body = await run_aws_signing(
self._sign_request,
service_name="bedrock",
headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
headers={
"Content-Type": "application/json",
"anthropic-version": MANTLE_ANTHROPIC_VERSION,
**({"anthropic-beta": ",".join(betas)} if betas else {}),
},
optional_params=litellm_params,
request_data=body,
api_base=endpoint_url,

View file

@ -7,7 +7,7 @@ to AWS Bedrock's CountTokens API format and vice versa.
import re
from collections.abc import Mapping
from typing import Final, Literal
from typing import Final, Literal, TypeAlias
from pydantic import JsonValue
@ -18,6 +18,14 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
# max_tokens field; CountTokens only counts input, so it has no effect
# on any generation.
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024
BedrockCountTokensInputType: TypeAlias = Literal["converse", "invokeModel"]
def _invoke_model_max_tokens(thinking: JsonValue) -> int:
budget: Final = thinking.get("budget_tokens") if isinstance(thinking, dict) else None
if not isinstance(budget, int):
return DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
return max(DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, budget + 1)
def _json_dict(value: JsonValue) -> dict[str, JsonValue]:
@ -72,7 +80,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
- Response: {"inputTokens": <number>}
"""
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> Literal["converse", "invokeModel"]:
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> BedrockCountTokensInputType:
"""
Detect whether to use 'converse' or 'invokeModel' input format.
@ -103,6 +111,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
def transform_anthropic_to_bedrock_count_tokens(
self,
request_data: Mapping[str, JsonValue],
input_type: BedrockCountTokensInputType | None = None,
) -> dict[str, JsonValue]:
"""
Transform request to Bedrock CountTokens format.
@ -133,9 +142,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
}
}
"""
input_type: Final = self._detect_input_type(request_data)
if input_type == "converse":
if (input_type or self._detect_input_type(request_data)) == "converse":
return self._transform_to_converse_format(request_data)
else:
return self._transform_to_invoke_model_format(request_data)
@ -196,7 +203,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
# Bedrock validates the body against the model's InvokeModel schema;
# Anthropic Messages bodies require these fields.
body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
body_data.setdefault("max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS)
body_data.setdefault("max_tokens", _invoke_model_max_tokens(body_data.get("thinking")))
# The CountTokens API expects invokeModel.body as a base64-encoded blob
encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode()
@ -257,23 +264,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
return {"input_tokens": input_tokens}
def validate_count_tokens_request(self, request_data: Mapping[str, JsonValue]) -> None:
def validate_count_tokens_request(
self, request_data: Mapping[str, JsonValue], input_type: BedrockCountTokensInputType | None = None
) -> None:
"""
Validate the incoming count tokens request.
Supports both Converse and InvokeModel input formats.
Args:
request_data: The request payload
Raises:
ValueError: If the request is invalid
"""
if not request_data.get("model"):
raise ValueError("model parameter is required")
input_type: Final = self._detect_input_type(request_data)
if input_type == "converse":
if (input_type or self._detect_input_type(request_data)) == "converse":
# Validate Converse format (messages-based)
messages: Final = request_data.get("messages", [])
if not messages:

View file

@ -1,4 +1,4 @@
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
@ -77,6 +77,7 @@ else:
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...])
class AmazonAnthropicClaudeMessagesConfig(
@ -100,6 +101,17 @@ class AmazonAnthropicClaudeMessagesConfig(
def beta_headers_provider(self) -> str:
return self.custom_llm_provider or "bedrock"
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return headers
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ())
def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
if bedrock_supports_per_turn_control(
model, _MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None))

View file

@ -18,6 +18,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
from litellm.llms.bedrock.common_utils import build_mantle_messages_url
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
@ -67,6 +68,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
def should_filter_anthropic_beta_headers(self) -> bool:
return False
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
return BaseAnthropicMessagesConfig.sent_anthropic_betas(self, headers, request)
def get_complete_url(
self,
api_base: str | None,

View file

@ -3,6 +3,7 @@ import datetime
import json
import math
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final
import httpx
@ -498,6 +499,8 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
import copy

View file

@ -3,6 +3,7 @@ OpenAI Token Counter implementation using the Responses API /input_tokens endpoi
"""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final
import openai
@ -96,6 +97,8 @@ class OpenAITokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using OpenAI's Responses API /input_tokens endpoint.

View file

@ -23,7 +23,7 @@ from litellm.types.llms.vertex_ai import (
PartType,
Schema,
)
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.utils import supports_response_schema, supports_system_messages
GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
@ -1259,8 +1259,6 @@ class VertexAITokenCounter(BaseTokenCounter):
self,
custom_llm_provider: str | None = None,
) -> bool:
from litellm.types.utils import LlmProviders
return custom_llm_provider == LlmProviders.VERTEX_AI.value
async def count_tokens(
@ -1275,6 +1273,8 @@ class VertexAITokenCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> TokenCountResponse | None:
import copy
@ -1313,6 +1313,24 @@ class VertexAITokenCounter(BaseTokenCounter):
# Check if this is a partner model (Claude, Mistral, etc.)
if VertexAIPartnerModels.is_vertex_partner_model(model_to_use):
from litellm.llms.anthropic.count_tokens.transformation import (
ChatCountTranslationError,
claude_count_options,
)
sent_messages: Final = messages or []
claude_options: Final = claude_count_options(
model_to_use,
LlmProviders.VERTEX_AI,
sent_messages,
tools,
request_options,
anthropic_beta,
deployment,
)
if isinstance(claude_options, ChatCountTranslationError):
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
# Use partner models token counter
partner_models_handler: Final = VertexAIPartnerModels()
@ -1336,13 +1354,19 @@ class VertexAITokenCounter(BaseTokenCounter):
try:
result = await partner_models_handler.count_tokens(
model=model_to_use,
messages=messages or [],
messages=sent_messages
if claude_options.messages is None
else [dict(message) for message in claude_options.messages],
litellm_params=partner_litellm_params,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_credentials=vertex_credentials,
system=system,
tools=tools,
tools=tools
if claude_options.messages is None
else [dict(tool) for tool in claude_options.tools or ()] or None,
count_options=claude_options.options,
betas=claude_options.betas,
)
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
status_code: Final = e.response.status_code

View file

@ -1,6 +1,8 @@
from collections.abc import Mapping, Sequence
from typing import Any, Final
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -17,6 +19,11 @@ from ....vertex_llm_base import VertexBase
from ..output_params_utils import sanitize_vertex_anthropic_output_params
def _is_web_search(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return isinstance(tool_type, str) and tool_type.startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value)
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
@property
def custom_llm_provider(self) -> str | None:
@ -66,59 +73,47 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
headers["content-type"] = "application/json"
# Add beta headers for Vertex AI
tools: Final = optional_params.get("tools", [])
beta_values: Final[set[str]] = set()
return dict(self.headers_with_anthropic_betas(headers, optional_params, messages)), api_base
# Get existing beta headers if any
existing_beta: Final = headers.get("anthropic-beta")
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
# Check for web search tool
for tool in tools:
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value)
break
# Check for tool search tools - Vertex AI uses different beta header
anthropic_model_info: Final = AnthropicModelInfo()
if anthropic_model_info.is_tool_search_used(tools):
beta_values.add(get_tool_search_beta_header("vertex_ai"))
if optional_params.get("safeguards") is not None:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
if beta_values:
headers["anthropic-beta"] = ",".join(beta_values)
return headers, api_base
def headers_with_anthropic_betas(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
tools: Final = tuple(
tool for tool in map(as_str_mapping, iter_items(optional_params.get("tools") or ())) if tool is not None
)
existing_betas: Final = (headers.get("anthropic-beta") or "").split(",")
beta_values: Final = frozenset(
(
*(beta.strip() for beta in existing_betas if beta.strip()),
*context_management_betas(optional_params.get("context_management")), # pyright: ignore[reportArgumentType] # optional_params is the config contract's untyped request body
*(
(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value,)
if any(map(_is_web_search, tools))
else ()
),
*(
(get_tool_search_beta_header("vertex_ai"),)
if AnthropicModelInfo().is_tool_search_used(list(tools))
else ()
),
*(
(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,)
if optional_params.get("safeguards") is not None
else ()
),
*(
(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value,)
if messages_carry_output_config(messages)
else ()
),
)
)
if not beta_values:
return headers
return {**headers, "anthropic-beta": ",".join(sorted(beta_values))}
def get_complete_url(
self,

View file

@ -6,6 +6,7 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Sequence
from typing import Any, Final
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
@ -91,6 +92,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
model: str,
request_data: dict[str, Any],
litellm_params: dict[str, Any],
betas: Sequence[str] = (),
) -> dict[str, Any]:
"""
Handle token counting request for a Vertex AI partner model.
@ -158,7 +160,10 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
)
# Prepare headers
headers: Final = {"Authorization": f"Bearer {access_token}"}
headers: Final = {
"Authorization": f"Bearer {access_token}",
**({"anthropic-beta": ",".join(betas)} if betas else {}),
}
# Get async HTTP client
from litellm import LlmProviders

View file

@ -1,6 +1,6 @@
# What is this?
## API Handler for calling Vertex AI Partner Models
from collections.abc import Callable
from collections.abc import Callable, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Final
@ -266,6 +266,8 @@ class VertexAIPartnerModels(VertexBase):
vertex_credentials=None,
system: object | None = None,
tools: list[dict[str, object]] | None = None,
count_options: Mapping[str, object] = MappingProxyType({}),
betas: Sequence[str] = (),
):
"""
Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
@ -302,6 +304,7 @@ class VertexAIPartnerModels(VertexBase):
**MappingProxyType(
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
),
**count_options,
}
# Prepare litellm_params with credentials
@ -319,6 +322,7 @@ class VertexAIPartnerModels(VertexBase):
model=model,
request_data=request_data,
litellm_params=_litellm_params,
betas=betas,
)
return result

View file

@ -50,7 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
with_litellm_call_id,
)
from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, should_auto_drop_params_for_agentic_cli
from litellm.router import Router
router: Final = APIRouter()
@ -333,12 +333,14 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format", "context_management")
class _CountTokensBodyOptions(TypedDict):
output_format: ReadOnly[JsonValue]
context_management: ReadOnly[JsonValue]
request_litellm_params: ReadOnly[Mapping[str, JsonValue]]
request_options: ReadOnly[Mapping[str, JsonValue]]
def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions:
@ -349,6 +351,9 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
"request_litellm_params": MappingProxyType(
{"drop_params": True, **sent_params} if auto_drop_params else sent_params
),
"request_options": MappingProxyType(
{key: data[key] for key in _COUNTED_REQUEST_OPTIONS if data.get(key) is not None}
),
}
@ -381,7 +386,7 @@ async def count_tokens(
Returns: {"input_tokens": <number>}
"""
from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config
from litellm.proxy.proxy_server import count_request_tokens, general_settings_view, llm_router, proxy_config
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
try:
@ -412,6 +417,11 @@ async def count_tokens(
request=token_request,
call_endpoint=True,
messages_format="anthropic",
anthropic_beta=request.headers.get("anthropic-beta"),
forward_client_headers=LiteLLMProxyRequestSetup.forwards_client_headers_for_all_models(
general_settings_view()
)
or LiteLLMProxyRequestSetup.forwards_client_headers_for_model_group(token_request.model, user_api_key_dict),
**_count_tokens_body_options(data, auto_drop_params),
)
except UnsupportedContentTypeError as e:

View file

@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
import litellm
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.count_tokens.transformation import is_request_rejection
from litellm.proxy._types import ProxyException
from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer
from litellm.types.utils import TokenCountResponse
@ -66,6 +67,10 @@ class ProviderCountFailure:
def cannot_count_model(self) -> bool:
return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES
@property
def rejected_request(self) -> bool:
return is_request_rejection(self.status_code)
def with_hint(self, hint: str) -> "ProviderCountFailure":
return replace(self, message=_with_hint(self.message, hint))

View file

@ -1527,27 +1527,33 @@ class LiteLLMProxyRequestSetup:
return returned_headers
@staticmethod
def add_headers_to_llm_call_by_model_group(data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth) -> dict:
"""
Add headers to the LLM call by model group
"""
def forwards_client_headers_for_all_models(general_settings: Mapping[str, object] | None) -> bool:
return general_settings is not None and general_settings.get("forward_client_headers_to_llm_api") is True
@staticmethod
def forwards_client_headers_for_model_group(model: str | None, user_api_key_dict: UserAPIKeyAuth) -> bool:
from litellm.proxy.auth.auth_checks import check_model_access_helper
from litellm.proxy.proxy_server import llm_router
data_model: Final = data.get("model")
if (
data_model is not None
return (
model is not None
and litellm.model_group_settings is not None
and litellm.model_group_settings.forward_client_headers_to_llm_api is not None
and check_model_access_helper(
model=data_model,
model=model,
llm_router=llm_router,
models=litellm.model_group_settings.forward_client_headers_to_llm_api,
team_model_aliases=user_api_key_dict.team_model_aliases,
team_id=user_api_key_dict.team_id,
) # handles aliases, wildcards, etc.
):
)
@staticmethod
def add_headers_to_llm_call_by_model_group(data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth) -> dict:
"""
Add headers to the LLM call by model group
"""
if LiteLLMProxyRequestSetup.forwards_client_headers_for_model_group(data.get("model"), user_api_key_dict):
_headers: Final = LiteLLMProxyRequestSetup.add_headers_to_llm_call(headers, user_api_key_dict)
if _headers != {}:
data["headers"] = _headers
@ -1584,7 +1590,7 @@ class LiteLLMProxyRequestSetup:
"""
data: Final = LitellmDataForBackendLLMCall()
if general_settings and general_settings.get("forward_client_headers_to_llm_api") is True:
if LiteLLMProxyRequestSetup.forwards_client_headers_for_all_models(general_settings):
_headers: Final = LiteLLMProxyRequestSetup.add_headers_to_llm_call(headers, user_api_key_dict)
if _headers != {}:
data["headers"] = _headers
@ -4038,14 +4044,19 @@ async def add_guardrails_from_policy_engine(
)
_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
(
LlmProviders.ANTHROPIC.value,
LlmProviders.BEDROCK.value,
LlmProviders.BEDROCK_MANTLE.value,
LlmProviders.VERTEX_AI.value,
)
_ANTHROPIC_API_HEADER_PROVIDER_NAMES: Final = (
LlmProviders.ANTHROPIC.value,
LlmProviders.BEDROCK.value,
LlmProviders.BEDROCK_MANTLE.value,
LlmProviders.VERTEX_AI.value,
)
_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(_ANTHROPIC_API_HEADER_PROVIDER_NAMES)
def receives_anthropic_api_headers(custom_llm_provider: str | None) -> bool:
return custom_llm_provider in _ANTHROPIC_API_HEADER_PROVIDER_NAMES
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))

View file

@ -293,7 +293,7 @@ def generate_feedback_box():
import contextlib
from collections import defaultdict
from contextlib import AsyncExitStack, asynccontextmanager
from functools import lru_cache, partial
from functools import lru_cache, partial, reduce
import litellm
import litellm._redis
@ -640,7 +640,11 @@ from litellm.proxy.list_api.common import (
problem_response,
request_validation_problem,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
add_litellm_data_to_request,
receives_anthropic_api_headers,
)
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
rust_control_plane_router,
)
@ -1316,6 +1320,8 @@ class _WebsearchInterceptionLitellmSettings(TypedDict, total=False):
_LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody)
_LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody)
_ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig)
_COUNT_REQUEST_OPTIONS: Final = TypeAdapter(Mapping[str, JsonValue])
_DROPPED_PARAM_PATHS: Final = TypeAdapter(tuple[object, ...])
_WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings)
_UNTYPED_VALUE: Final = TypeAdapter(object)
@ -14279,6 +14285,8 @@ async def _try_provider_token_count(
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
@ -14305,6 +14313,8 @@ async def _try_provider_token_count(
messages_format=messages_format,
output_format=output_format,
context_management=context_management,
request_options=request_options,
anthropic_beta=anthropic_beta,
)
except httpx.HTTPStatusError as e:
return ProviderCountFailure(
@ -14358,6 +14368,33 @@ async def _provider_token_count(
raise failure.to_proxy_exception()
def _retried_without_rejected_options(
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
request_options: Mapping[str, JsonValue],
) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]:
if not request_options:
return count_with
return partial(_count_retrying_without_options, count_with, tuple(request_options))
async def _count_retrying_without_options(
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
forwarded_options: tuple[str, ...],
model: str,
region: str | None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
counted: Final = await count_with(model, region)
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
return counted
verbose_proxy_logger.warning(
"Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",
model,
"could not be built" if counted.rejected_before_provider_call else f"answered {counted.status_code}",
", ".join(forwarded_options),
)
return await count_with(model, region, request_options=MappingProxyType({}))
@dataclass(frozen=True, slots=True)
class _LocalTokenEstimate:
total_tokens: int
@ -14682,6 +14719,33 @@ def _count_context_management(
)
_STRING_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object])
def _deployment_with_request_drop_params(
deployment: Mapping[str, object] | None, sent_params: Mapping[str, object]
) -> Mapping[str, object] | None:
if "drop_params" not in sent_params:
return deployment
resolved_deployment: Final = deployment or {}
litellm_params: Final = _STRING_KEYED_MAPPING.validate_python(resolved_deployment.get("litellm_params") or {})
return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": sent_params["drop_params"]}}
def _count_request_options(
request_options: Mapping[str, JsonValue],
deployment: Mapping[str, object] | None,
sent_params: Mapping[str, object],
) -> Mapping[str, JsonValue]:
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
deployment_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
dropped: Final = {**deployment_params, **sent_params}.get("additional_drop_params")
dropped_values: Final = _DROPPED_PARAM_PATHS.validate_python(dropped) if isinstance(dropped, list) else ()
dropped_paths: Final = tuple(path for path in dropped_values if isinstance(path, str))
return _COUNT_REQUEST_OPTIONS.validate_python(reduce(delete_nested_value, dropped_paths, dict(request_options)))
def _system_message(system: object) -> ChatCompletionSystemMessage | None:
if not isinstance(system, (str, list)) or not system:
return None
@ -14714,6 +14778,9 @@ async def count_request_tokens(
output_format: JsonValue = None,
context_management: JsonValue = None,
request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}),
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
forward_client_headers: bool = False,
) -> TokenCountResponse:
prompt: Final = request.prompt
messages: Final = request.messages
@ -14781,23 +14848,40 @@ async def count_request_tokens(
prompt_only: Final = bool(prompt) and conversation_empty
contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents))
prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES
sent_params: Final[Mapping[str, object]] = {
**(_router_default_litellm_params(llm_router) if llm_router is not None else {}),
**request_litellm_params,
}
counted_options: Final = _count_request_options(request_options, deployment_view, sent_params)
ladder: Final = (
None
if provider_counter is None
else await _provider_token_count(
count_with=partial(
_try_provider_token_count,
provider_counter,
custom_llm_provider,
messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages,
contents=None if prompt_as_message else contents,
deployment=deployment,
request_model=request.model,
tools=tools,
system=system,
messages_format=messages_format,
output_format=output_format,
context_management=_count_context_management(context_management, request.model, request_litellm_params),
count_with=_retried_without_rejected_options(
partial(
_try_provider_token_count,
provider_counter,
custom_llm_provider,
messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages,
contents=None if prompt_as_message else contents,
deployment=_deployment_with_request_drop_params(
None if deployment is None else _STRING_KEYED_MAPPING.validate_python(deployment),
sent_params,
),
request_model=request.model,
tools=tools,
system=system,
messages_format=messages_format,
output_format=output_format,
context_management=_count_context_management(
context_management, request.model, request_litellm_params
),
request_options=counted_options,
anthropic_beta=anthropic_beta
if forward_client_headers or receives_anthropic_api_headers(custom_llm_provider)
else None,
),
counted_options,
),
model_to_use=model_to_use,
settings=settings,

View file

@ -1,7 +1,10 @@
import asyncio
from base64 import b64encode
from collections.abc import Mapping
from copy import deepcopy
from queue import SimpleQueue
from threading import get_ident
from types import MappingProxyType
from typing import Final
import httpx
@ -10,16 +13,25 @@ import respx
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
)
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@pytest.mark.parametrize(
@ -441,3 +453,337 @@ async def test_handler_sends_the_leading_system_run_as_system_not_as_a_message(h
"system": [{"type": "text", "text": "Be terse"}],
"messages": [{"role": "user", "content": "Hello, how are you?"}],
}
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {"type": "object", "properties": {"city": {"type": "string"}}}
_ANSWER_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
"additionalProperties": False,
}
_COUNTERS: Final = pytest.mark.parametrize(
("counter_type", "count_endpoint"),
(
(AnthropicTokenCounter, "https://gateway.example/v1/messages/count_tokens"),
(AzureAIAnthropicTokenCounter, "https://gateway.example/anthropic/v1/messages/count_tokens"),
),
)
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
async def _counted_requests(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
messages: list[dict[str, JsonValue]],
tools: list[dict[str, JsonValue]] | None,
messages_format: TokenCountMessagesFormat,
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
model: str = "claude-opus-5-5",
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def count(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(200, json={"input_tokens": 31})
with respx.mock:
respx.post(count_endpoint).mock(side_effect=count)
result: Final = await counter_type().count_tokens(
model_to_use=model,
messages=messages,
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
request_model=model,
tools=tools,
messages_format=messages_format,
request_options=request_options,
anthropic_beta=anthropic_beta,
)
return result, tuple(sent.get() for _ in range(sent.qsize()))
_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
_INFERENCE_PROVIDERS: Final = MappingProxyType(
{AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"}
)
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}
_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000}
async def _messages_inference_request(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
inference_endpoint: str,
request_options: Mapping[str, JsonValue],
model: str = "claude-haiku-5-5",
extra_headers: Mapping[str, str] = MappingProxyType({}),
) -> httpx.Request:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def reply(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(200, json=_MESSAGE_REPLY)
with respx.mock:
respx.post(inference_endpoint).mock(side_effect=reply)
await litellm.anthropic_messages(
model=f"{_INFERENCE_PROVIDERS[counter_type]}/{model}",
messages=[{"role": "user", "content": "Weather in SF?"}],
tools=[_WEATHER_TOOL],
max_tokens=4096,
api_key="sk-ant-api03-test-key",
api_base="https://gateway.example",
extra_headers=dict(extra_headers),
**request_options,
)
return sent.get()
def _on_both_counters(
*betas: str,
) -> Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]]:
return MappingProxyType({AnthropicTokenCounter: frozenset(betas), AzureAIAnthropicTokenCounter: frozenset(betas)})
def _betas(request: httpx.Request) -> frozenset[str]:
return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
@_COUNTERS
@pytest.mark.parametrize(
("request_options", "expected_betas"),
(
({"thinking": _ENABLED_THINKING}, _on_both_counters()),
({"tool_choice": {"type": "any"}}, _on_both_counters()),
(
{"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}},
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value),
),
(
{
"thinking": _ENABLED_THINKING,
"context_management": {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
},
},
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value),
),
(
{"context_management": {"edits": [{"type": "compact_20260112"}]}},
MappingProxyType(
{
AnthropicTokenCounter: frozenset((ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,)),
AzureAIAnthropicTokenCounter: frozenset(),
}
),
),
),
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
)
async def test_counter_sends_the_options_and_context_edits_messages_inference_sends(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
request_options: dict[str, JsonValue],
expected_betas: Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]],
) -> None:
inference: Final = await _messages_inference_request(
counter_type, count_endpoint.removesuffix("/count_tokens"), request_options
)
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
request_options=request_options,
model="claude-haiku-5-5",
)
inference_body: Final = _JSON_BODY.validate_json(inference.content)
count_body: Final = _JSON_BODY.validate_json(count.content)
inference_output_format: Final = inference_body.get("output_format")
inference_output_config: Final = inference_body.get("output_config")
assert "output_format" not in count_body
assert count_body.get("output_config") == (
inference_output_config
if inference_output_format is None
else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format}
)
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
}
assert expected_betas[counter_type] == _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_sends_the_thinking_display_and_caller_betas_messages_inference_sends(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
caller_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
display_thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
inference: Final = await _messages_inference_request(
counter_type,
count_endpoint.removesuffix("/count_tokens"),
display_thinking,
model="claude-opus-4-6",
extra_headers={"anthropic-beta": caller_beta},
)
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
request_options=display_thinking,
anthropic_beta=caller_beta,
model="claude-opus-4-6",
)
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
assert counted_betas == _betas(inference)
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_without_options_sends_only_the_token_counting_beta_and_the_bare_count_body(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
anthropic_beta=ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value,
model="claude-haiku-5-5",
)
assert _betas(count) == {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
assert _JSON_BODY.validate_json(count.content) == {
"model": "claude-haiku-5-5",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"tools": [_WEATHER_TOOL],
}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_sending_options_still_counts_the_leading_system_run_as_system(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "system", "content": "Be terse."}, {"role": "user", "content": "Weather in SF?"}],
None,
"anthropic",
request_options={"thinking": _ENABLED_THINKING},
model="claude-haiku-5-5",
)
body: Final = _JSON_BODY.validate_json(count.content)
assert "thinking" in body
assert {name: body.get(name) for name in ("system", "messages")} == {
"system": [{"type": "text", "text": "Be terse."}],
"messages": [{"role": "user", "content": "Weather in SF?"}],
}
_ADVISOR_TOOL: Final[dict[str, JsonValue]] = {
"type": ANTHROPIC_ADVISOR_TOOL_TYPE,
"name": "advisor",
"model": "claude-opus-5-5",
}
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}}
_ADVISOR_RESULT: Final[dict[str, JsonValue]] = {
"type": "advisor_tool_result",
"tool_use_id": "srvtoolu_1",
"content": {"type": "advisor_result", "text": "Check the forecast first."},
}
_ADVISED_ANSWER: Final[dict[str, JsonValue]] = {"type": "text", "text": "It is sunny."}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
@pytest.mark.parametrize(
("tools", "counted_reply"),
[
pytest.param(
[_ADVISOR_TOOL, _WEATHER_TOOL],
[_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER],
id="advisor_tool_sent",
),
pytest.param([_WEATHER_TOOL], [_ADVISED_ANSWER], id="advisor_tool_not_sent"),
],
)
async def test_counter_sending_options_keeps_advisor_blocks_only_when_the_advisor_tool_is_sent(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
tools: list[dict[str, JsonValue]],
counted_reply: list[JsonValue],
) -> None:
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[
{"role": "user", "content": "Weather in SF?"},
{"role": "assistant", "content": [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER]},
{"role": "user", "content": "And tomorrow?"},
],
tools,
"anthropic",
request_options={"thinking": _ENABLED_THINKING},
model="claude-haiku-5-5",
)
body: Final = _JSON_BODY.validate_json(count.content)
assert {name: body.get(name) for name in ("messages", "tools")} == {
"messages": [
{"role": "user", "content": "Weather in SF?"},
{"role": "assistant", "content": counted_reply},
{"role": "user", "content": "And tomorrow?"},
],
"tools": tools,
}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_refuses_options_messages_inference_refuses_without_calling_anthropic(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
result, requests = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
None,
"anthropic",
request_options={"output_config": {"effort": "fastest"}},
model="claude-haiku-4-5",
)
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
assert requests == ()

View file

@ -1,5 +1,8 @@
import base64
import json
from typing import Final
import pytest
from litellm.llms.bedrock.count_tokens.transformation import (
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS,
@ -53,6 +56,28 @@ def test_detect_input_type_converse_blocks_route_to_converse():
assert config._detect_input_type(request) == "converse"
@pytest.mark.parametrize(
"option",
(
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
{"tool_choice": {"type": "auto"}},
{"output_config": {"effort": "low"}},
),
)
def test_pass_through_body_with_string_content_is_counted_as_converse_whatever_options_it_carries(
option: dict[str, object],
) -> None:
request: Final = {
"model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
"messages": [{"role": "user", "content": "hi"}],
**option,
}
result: Final = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request)
assert result["input"] == {"converse": {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}}
def test_transform_to_invoke_model_format_base64_encodes_body():
"""The CountTokens API expects invokeModel.body as a base64-encoded blob;
Anthropic Messages bodies additionally need anthropic_version/max_tokens

View file

@ -1,12 +1,20 @@
from collections.abc import Mapping
import base64
import json
from collections.abc import Callable, Mapping
from typing import Final
import httpx
import pytest
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_EFFORT_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
)
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
@ -251,3 +259,500 @@ async def test_explicit_mantle_failure_does_not_probe_runtime() -> None:
assert result.status_code == 404
assert result.tokenizer_type == "bedrock_mantle_api"
assert bedrock.posted_hosts() == (MANTLE_HOST,)
def _invoke_body(request: httpx.Request) -> dict[str, JsonValue]:
invoke_model: Final = _JSON_BODY.validate_json(request.content)["input"]
assert isinstance(invoke_model, dict)
encoded: Final = invoke_model["invokeModel"]
assert isinstance(encoded, dict) and isinstance(encoded["body"], str)
return _JSON_BODY.validate_json(base64.b64decode(encoded["body"]))
_CLAUDE_REPLY: Final = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
}
_THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
"thinking": {"type": "enabled", "budget_tokens": 2000},
"tool_choice": {"type": "auto"},
}
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
_CITY_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
async def _messages_inference_and_count(
deployment_model: str,
model_to_use: str,
request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS,
tool: Mapping[str, JsonValue] = _WEATHER_TOOL,
runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}),
inference_reply: Mapping[str, object] = _CLAUDE_REPLY,
) -> tuple[_Bedrock, _Bedrock]:
credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, inference_reply), mantle=(200, inference_reply))
await litellm.anthropic_messages(
model=deployment_model,
messages=MESSAGES,
tools=[dict(tool)],
max_tokens=4096,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**request_options,
**credentials,
)
bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use=model_to_use,
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}},
request_model="claude",
tools=[dict(tool)],
request_options=request_options,
)
assert result is not None
assert result.error is False
return inference, bedrock
def _betas_besides_effort(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]:
betas: Final = body.get("anthropic_beta") or []
assert isinstance(betas, list)
return tuple(beta for beta in betas if beta != ANTHROPIC_EFFORT_BETA_HEADER)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "model_to_use"),
(
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-sonnet-4-6"),
("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"),
),
)
async def test_runtime_count_sends_the_options_and_context_edits_invoke_inference_sends(
deployment_model: str, model_to_use: str
) -> None:
inference, bedrock = await _messages_inference_and_count(deployment_model, model_to_use)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _invoke_body(bedrock.requests[0])
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
assert _betas_besides_effort(counted) == _betas_besides_effort(sent)
thinking: Final = counted["thinking"]
assert isinstance(thinking, dict)
budget_tokens: Final = thinking.get("budget_tokens", 0)
assert isinstance(budget_tokens, int) and isinstance(counted["max_tokens"], int)
assert counted["max_tokens"] > budget_tokens
@pytest.mark.asyncio
async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5"
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
assert "anthropic_beta" not in counted
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
_DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
return tuple(sorted(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "request_options", "expected_beta"),
(
(
"bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
{"output_config": {"effort": "low"}},
ANTHROPIC_EFFORT_BETA_HEADER,
),
("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER),
),
ids=("effort", "thinking_display"),
)
async def test_runtime_count_sends_the_betas_invoke_inference_sends(
deployment_model: str, request_options: dict[str, JsonValue], expected_beta: str
) -> None:
inference, bedrock = await _messages_inference_and_count(
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta")
assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
assert isinstance(counted_betas, list) and expected_beta in counted_betas
_EFFORT: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}}
_CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = {
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]}
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_options", "expected_betas"),
((_DISPLAY_THINKING, (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)), (_EFFORT, ())),
ids=("thinking_display", "effort"),
)
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends(
request_options: dict[str, JsonValue], expected_betas: tuple[str, ...]
) -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/invoke/us.anthropic.claude-opus-5-5",
"us.anthropic.claude-opus-5-5",
request_options,
runtime=(400, UNSUPPORTED),
)
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", [])
assert isinstance(sent_betas, list)
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) == expected_betas
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "request_options"),
(
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _EFFORT),
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLEAR_THINKING_EDITS),
("bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", _EFFORT),
),
ids=("effort", "clear_thinking_sets_effort", "inference_sends_the_effort_beta"),
)
async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_betas_invoke_inference_sends(
deployment_model: str, request_options: dict[str, JsonValue]
) -> None:
inference, bedrock = await _messages_inference_and_count(
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _invoke_body(bedrock.requests[0])
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
counted_config: Final = counted.get("output_config")
assert isinstance(counted_config, dict) and counted_config.get("effort") == "low"
counted_betas: Final = counted.get("anthropic_beta")
sent_betas: Final = sent.get("anthropic_beta", [])
assert isinstance(counted_betas, list) and isinstance(sent_betas, list)
assert counted_betas.count(ANTHROPIC_EFFORT_BETA_HEADER) == 1
assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER}
@pytest.mark.asyncio
async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/invoke/us.anthropic.claude-sonnet-4-6",
"us.anthropic.claude-sonnet-4-6",
{**_CITY_FORMAT, "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _invoke_body(bedrock.requests[0])
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
assert counted.get("output_config") == sent.get("output_config")
assert counted.get("anthropic_beta") == sent.get("anthropic_beta")
_CONVERSE_REPLY: Final = {
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4},
"metrics": {"latencyMs": 1},
}
_NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = {
"tool_choice": {"type": "auto", "disable_parallel_tool_use": True}
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "inference_reply"),
(
("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY),
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY),
),
ids=("converse", "invoke"),
)
async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it(
deployment_model: str, inference_reply: Mapping[str, object]
) -> None:
inference, bedrock = await _messages_inference_and_count(
deployment_model,
deployment_model.split("/")[-1],
_NO_PARALLEL_TOOL_USE,
inference_reply=inference_reply,
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted_tool_choice: Final = _invoke_body(bedrock.requests[0]).get("tool_choice")
assert isinstance(counted_tool_choice, dict) and counted_tool_choice.get("type") == "auto"
assert ("disable_parallel_tool_use" in counted_tool_choice) == (
b"disable_parallel_tool_use" in inference.requests[0].content
)
@pytest.mark.asyncio
async def test_count_on_a_stand_in_model_counts_the_messages_and_output_config_the_deployment_model_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/us.anthropic.claude-haiku-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _CITY_FORMAT
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
fields: Final = ("messages", "output_config")
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
@pytest.mark.asyncio
async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
assert _sent_betas(bedrock.requests[0]) == _sent_betas(inference.requests[0])
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0])
def _mantle_answers(status_code: int, body: Mapping[str, JsonValue]) -> Callable[[httpx.Request], httpx.Response]:
return lambda request: httpx.Response(status_code, json=dict(body), request=request)
def _mantle_unreachable(error: type[httpx.TransportError]) -> Callable[[httpx.Request], httpx.Response]:
def unreachable(request: httpx.Request) -> httpx.Response:
raise error("bedrock-mantle is unreachable", request=request)
return unreachable
_MANTLE_REJECTION: Final[dict[str, JsonValue]] = {
"type": "error",
"error": {"type": "invalid_request_error", "message": "thinking: unsupported"},
}
_MANTLE_FAILURE: Final[dict[str, JsonValue]] = {"type": "error", "error": {"type": "api_error", "message": "down"}}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_options", "mantle", "expected"),
(
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ConnectError), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ReadTimeout), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(503, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(429, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(403, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(400, _MANTLE_REJECTION), (True, 400, "bedrock_mantle_api")),
(_DISPLAY_THINKING, _mantle_answers(422, _MANTLE_REJECTION), (True, 422, "bedrock_mantle_api")),
(_DISPLAY_THINKING, _mantle_answers(200, {"input_tokens": 2177}), (False, None, "bedrock_mantle_api")),
({}, _mantle_answers(503, _MANTLE_FAILURE), (True, 503, "bedrock_mantle_api")),
),
ids=(
"mantle-unreachable",
"mantle-timeout",
"mantle-5xx",
"mantle-rate-limit",
"mantle-denied",
"mantle-rejects-the-options",
"mantle-rejects-the-body",
"mantle-counts",
"no-options-mantle-5xx",
),
)
async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or_rejects_the_request(
request_options: dict[str, JsonValue],
mantle: Callable[[httpx.Request], httpx.Response],
expected: tuple[bool, int | None, str],
) -> None:
def bedrock(request: httpx.Request) -> httpx.Response:
if request.url.host == RUNTIME_HOST:
return httpx.Response(400, json=UNSUPPORTED, request=request)
return mantle(request)
result: Final = await BedrockTokenCounter(
client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock))
).count_tokens(
model_to_use="us.anthropic.claude-opus-5-5",
messages=MESSAGES,
contents=None,
deployment={
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"}
},
request_model="claude-opus-5-5",
request_options=request_options,
)
assert result is not None
assert (result.error, result.status_code, result.tokenizer_type) == expected
_CALLER_BETAS: Final = (ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value, "context-1m-2025-08-07")
@pytest.mark.asyncio
async def test_runtime_count_sends_the_caller_betas_invoke_inference_keeps_for_the_model() -> None:
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0"
model_info: Final[dict[str, JsonValue]] = {"supports_per_turn_control": True}
thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}}
credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages(
model=deployment_model,
messages=MESSAGES,
max_tokens=4096,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
extra_headers={"anthropic-beta": ",".join(_CALLER_BETAS)},
model_info=model_info,
**thinking,
**credentials,
)
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use=deployment_model.removeprefix("bedrock/invoke/"),
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
request_model="claude",
request_options=thinking,
anthropic_beta=",".join(_CALLER_BETAS),
)
assert result is not None
assert result.error is False
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta")
assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
assert isinstance(counted_betas, list) and set(_CALLER_BETAS) <= set(counted_betas)
@pytest.mark.asyncio
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
"bedrock/invoke/us.anthropic.claude-opus-5-5",
"us.anthropic.claude-opus-5-5",
_THINKING_AND_TOOL_CHOICE,
tool={**_WEATHER_TOOL, "eager_input_streaming": True},
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
def _texts(message: JsonValue) -> tuple[str, ...]:
content: Final = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
return ()
return tuple(text for block in content if isinstance(block, dict) and isinstance(text := block.get("text"), str))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("native_output", "request_options", "schema_inlined"),
(
(False, _CITY_FORMAT, True),
(True, _CITY_FORMAT, False),
(False, _THINKING_AND_TOOL_CHOICE, False),
),
ids=("inlined_schema", "native_schema", "no_format"),
)
async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends(
native_output: bool, request_options: dict[str, JsonValue], schema_inlined: bool
) -> None:
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-count-probe-v1:0"
model_info: Final = {"supports_bedrock_native_structured_output": native_output}
credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages(
model=deployment_model,
messages=MESSAGES,
max_tokens=4096,
model_info=model_info,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**request_options,
**credentials,
)
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-count-probe-v1:0",
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
request_model="claude",
request_options=request_options,
)
assert result is not None
assert (result.error, result.total_tokens) == (False, 412)
counted: Final = _invoke_body(bedrock.requests[0])["messages"]
assert counted == _JSON_BODY.validate_json(inference.requests[0].content)["messages"]
assert isinstance(counted, list)
assert (json.dumps(_CITY_SCHEMA) in _texts(counted[-1])) is schema_inlined
assert (counted == MESSAGES) is not schema_inlined
_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("messages", "tools", "request_options"),
(
(MESSAGES, None, {"output_config": {"effort": "fastest"}}),
(MESSAGES, [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], _THINKING),
(MESSAGES, ["get_weather"], _THINKING),
([{"role": "user", "content": 5}], None, _THINKING),
([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING),
),
ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"),
)
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock(
messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue]
) -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
messages=messages,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-sonnet-4-5",
tools=tools,
request_options=request_options,
)
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
assert bedrock.posted_hosts() == ()

View file

@ -1,7 +1,9 @@
import copy
import time
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, patch
@ -28,6 +30,7 @@ from litellm.llms.vertex_ai.common_utils import (
vertex_request_labels_from_litellm_params,
)
from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
@ -2291,3 +2294,179 @@ async def test_vertex_ai_token_counter_routes_like_generate_content(
)
assert (None if result is None else result.total_tokens) == vertex_counted_tokens
_PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [
{"role": "user", "content": "Weather in SF?"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
_PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = {
"type": "function",
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
}
async def _partner_counted_bodies(
model: str, request_options: Mapping[str, JsonValue] = MappingProxyType({})
) -> tuple[dict[str, object], ...]:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
class FakeResponse:
status_code = 200
def json(self) -> dict[str, int]:
return {"input_tokens": 37}
class FakeHttpClient:
posted_bodies: tuple[dict[str, object], ...] = ()
async def post(
self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float
) -> FakeResponse:
self.posted_bodies = (*self.posted_bodies, json)
return FakeResponse()
fake_http_client: Final = FakeHttpClient()
with (
patch.object(handler, "get_async_httpx_client", return_value=fake_http_client),
patch.object(
handler.VertexAIPartnerModelsTokenCounter,
"_ensure_access_token_async",
new=AsyncMock(return_value=("fake-token", "test-project")),
),
):
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use=model,
messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS),
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=[_PARTNER_RESPONSES_TOOL],
request_options=request_options,
)
assert result is not None
assert result.total_tokens == 37
return fake_http_client.posted_bodies
_PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
async def _vertex_claude_inference_and_count(
tools: list[dict[str, JsonValue]], request_options: Mapping[str, JsonValue]
) -> tuple[httpx.Request, httpx.Request]:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def vertex(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(
200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY
)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex))
with (
patch.object(handler, "get_async_httpx_client", return_value=client),
patch.object(
VertexBase, "_ensure_access_token_async", new=AsyncMock(return_value=("fake-token", "test-project"))
),
patch.object(VertexBase, "_ensure_access_token", return_value=("fake-token", "test-project")),
):
await litellm.anthropic_messages(
model="vertex_ai/claude-haiku-5-5",
messages=messages,
tools=tools,
max_tokens=4096,
vertex_project="test-project",
vertex_location="us-east5",
client=client,
**request_options,
)
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use="claude-haiku-5-5",
messages=messages,
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=tools,
request_options=request_options,
)
assert result is not None
assert result.total_tokens == 37
inference, count = tuple(sent.get() for _ in range(sent.qsize()))
assert count.url.path.endswith("count-tokens:rawPredict")
return inference, count
@pytest.mark.asyncio
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
inference, count = await _vertex_claude_inference_and_count(
[{"name": "get_weather", "input_schema": {"type": "object"}}],
{
"thinking": {"type": "enabled", "budget_tokens": 2000},
"tool_choice": {"type": "auto"},
"context_management": {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
},
},
)
counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content)
inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
assert {name: counted.get(name) for name in fields} == {name: inferred.get(name) for name in fields}
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
@pytest.mark.asyncio
async def test_vertex_claude_count_sends_the_tool_search_beta_vertex_messages_inference_sends() -> None:
inference, count = await _vertex_claude_inference_and_count(
[
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
{"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True},
],
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
)
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
assert get_tool_search_beta_header("vertex_ai") in count.headers["anthropic-beta"].split(",")
@pytest.mark.asyncio
async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None:
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use="claude-haiku-4-5",
messages=[{"role": "user", "content": "Weather in SF?"}],
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
request_options={"output_config": {"effort": "fastest"}},
)
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
@pytest.mark.asyncio
async def test_vertex_llama_count_leaves_out_the_claude_only_options() -> None:
model: Final = "meta/llama-4-maverick-17b-128e-instruct-maas"
assert await _partner_counted_bodies(
model, request_options={"thinking": {"type": "enabled", "budget_tokens": 2000}}
) == await _partner_counted_bodies(model)

View file

@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import anyio
import httpx
import pytest
import respx
import tiktoken
from dotenv import load_dotenv
from pydantic import BaseModel, ConfigDict, JsonValue
@ -36,6 +37,7 @@ load_dotenv()
from fastapi import Request
from fastapi.responses import JSONResponse
from starlette.types import Message
import litellm
from litellm import Router
@ -53,10 +55,11 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import (
kimi_k3_tokens,
load_kimi_k3_encoding,
)
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensInputType
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter
from litellm.proxy._types import ProxyException, TokenCountRequest
from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth
from litellm.proxy.anthropic_endpoints.endpoints import (
count_tokens as anthropic_count_tokens,
)
@ -76,6 +79,8 @@ from litellm.proxy.proxy_server import (
token_counter,
)
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.router import CustomRoutingStrategyBase
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
from litellm.types.utils import SelectTokenizerResponse, TokenCountMessagesFormat, TokenCountResponse
@ -240,6 +245,9 @@ async def test_anthropic_messages_count_tokens_endpoint():
output_format=None,
context_management=None,
request_litellm_params=None,
request_options=None,
anthropic_beta=None,
forward_client_headers=False,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "claude-3-sonnet-20240229"
@ -320,6 +328,9 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
output_format=None,
context_management=None,
request_litellm_params=None,
request_options=None,
anthropic_beta=None,
forward_client_headers=False,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "gpt-4"
@ -809,6 +820,7 @@ class _FailingRuntimeHandler(BedrockCountTokensHandler):
litellm_params: dict[str, object],
resolved_model: str,
client: AsyncHTTPHandler | None = None,
input_type: BedrockCountTokensInputType | None = None,
) -> dict[str, JsonValue]:
raise self._error
@ -1138,6 +1150,9 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
output_format=None,
context_management=None,
request_litellm_params=None,
request_options=None,
anthropic_beta=None,
forward_client_headers=False,
):
raise ProxyException(
message='{"detail":{"message":"Input is too long for requested model."}}',
@ -1195,6 +1210,9 @@ async def test_anthropic_endpoint_403_permission_error_format():
output_format=None,
context_management=None,
request_litellm_params=None,
request_options=None,
anthropic_beta=None,
forward_client_headers=False,
):
raise ProxyException(
message='{"Message":"Bearer Token has expired"}',
@ -1252,6 +1270,9 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
output_format=None,
context_management=None,
request_litellm_params=None,
request_options=None,
anthropic_beta=None,
forward_client_headers=False,
):
raise ProxyException(
message="Rate limit exceeded",
@ -1276,6 +1297,660 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
proxy_server.count_request_tokens = original_token_counter
def _json_request(
body: Mapping[str, object], user_agent: str, headers: Mapping[str, str] = MappingProxyType({})
) -> Request:
payload: Final = json.dumps(body).encode()
async def receive() -> Message:
return {"type": "http.request", "body": payload, "more_body": False}
return Request(
{
"type": "http",
"method": "POST",
"path": "/v1/messages/count_tokens",
"query_string": b"",
"headers": [
(b"content-type", b"application/json"),
(b"user-agent", user_agent.encode()),
*((name.encode(), value.encode()) for name, value in headers.items()),
],
},
receive,
)
_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = {
"thinking": {"type": "adaptive"},
"tool_choice": {"type": "auto"},
"output_config": {
"effort": "low",
"format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
},
"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
}
def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
output_config: Final = body.get("output_config")
return frozenset(name for name in _CLAUDE_COUNT_OPTIONS if name in body) | frozenset(
f"output_config.{key}" for key in (output_config if isinstance(output_config, dict) else {})
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("additional_drop_params", "request_params", "expected_paths"),
(
(
None,
{},
{
"thinking",
"tool_choice",
"output_config",
"output_config.effort",
"output_config.format",
"context_management",
},
),
(
["tool_choice", "output_config.format"],
{},
{"thinking", "output_config", "output_config.effort", "context_management"},
),
(
["context_management"],
{},
{"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"},
),
(
None,
{"additional_drop_params": ["thinking", "context_management"]},
{"tool_choice", "output_config", "output_config.effort", "output_config.format"},
),
),
ids=(
"nothing-dropped",
"deployment-drops-tool-choice-and-the-format",
"deployment-drops-context-management",
"request-drops-thinking-and-context-management",
),
)
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
monkeypatch: pytest.MonkeyPatch,
additional_drop_params: list[str] | None,
request_params: dict[str, JsonValue],
expected_paths: set[str],
) -> None:
import litellm.proxy.proxy_server as proxy_server
router: Final = Router(
model_list=[
{
"model_name": "claude-agent",
"litellm_params": {
"model": "anthropic/claude-haiku-5-5",
"api_key": "sk-ant-test",
"additional_drop_params": additional_drop_params,
},
}
]
)
request: Final = _json_request(
{
"model": "claude-agent",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
**_CLAUDE_COUNT_OPTIONS,
**request_params,
},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 7})
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
assert response["input_tokens"] == 7
sent_request: Final = route.calls.last.request
assert _sent_option_paths(json.loads(sent_request.content)) == expected_paths
assert ("context-management-2025-06-27" in sent_request.headers["anthropic-beta"]) == (
"context_management" in expected_paths
)
_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
def _claude_router(deployment_model: str, deployment_params: Mapping[str, JsonValue]) -> Router:
return Router(
model_list=[
{
"model_name": "claude-agent",
"litellm_params": {"model": deployment_model, "api_key": "sk-ant-test", **deployment_params},
}
]
)
async def _count_on_a_claude_deployment(
monkeypatch: pytest.MonkeyPatch,
deployment_model: str,
body: Mapping[str, JsonValue],
reply: Callable[[httpx.Request], httpx.Response],
user_agent: str = "python-httpx/0.28.1",
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
) -> tuple[object, tuple[httpx.Request, ...]]:
import litellm.proxy.proxy_server as proxy_server
router: Final = _claude_router(deployment_model, deployment_params)
request: Final = _json_request(
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
user_agent,
{"anthropic-beta": _CALLER_BETA},
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(side_effect=reply)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
return response, tuple(call.request for call in route.calls)
def _counts_only_without_options(request: httpx.Request) -> httpx.Response:
if _sent_option_paths(json.loads(request.content)):
return httpx.Response(
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}}
)
return httpx.Response(200, json={"input_tokens": 7})
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "options", "rejected_by_anthropic"),
(
("anthropic/claude-haiku-4-5", {"output_config": {"effort": "fastest"}}, False),
("anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, True),
),
ids=("rejected-by-the-count-transform", "rejected-by-anthropic"),
)
async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_claude_count_rejects_them(
monkeypatch: pytest.MonkeyPatch,
deployment_model: str,
options: dict[str, JsonValue],
rejected_by_anthropic: bool,
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch, deployment_model, options, _counts_only_without_options
)
assert not isinstance(response, JSONResponse)
assert response["input_tokens"] == 7
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
*((True,) if rejected_by_anthropic else ()),
False,
]
assert [_CALLER_BETA in sent.headers.get("anthropic-beta", "") for sent in requests] == [
*((True,) if rejected_by_anthropic else ()),
False,
]
@pytest.mark.asyncio
@pytest.mark.parametrize(("status_code", "counted_again"), ((401, False), (403, False), (404, True), (422, True)))
async def test_anthropic_count_tokens_counts_again_only_when_the_claude_count_rejects_the_request(
monkeypatch: pytest.MonkeyPatch, status_code: int, counted_again: bool
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-haiku-5-5",
_CLAUDE_COUNT_OPTIONS,
lambda _: httpx.Response(status_code, json={"type": "error", "error": {"type": "error", "message": "denied"}}),
)
assert isinstance(response, JSONResponse)
assert response.status_code == status_code
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
True,
*((False,) if counted_again else ()),
]
@pytest.mark.asyncio
async def test_anthropic_count_tokens_warns_with_every_option_it_counts_again_without(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
verbose_proxy_logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
await _count_on_a_claude_deployment(
monkeypatch, "anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, _counts_only_without_options
)
finally:
verbose_proxy_logger.removeHandler(caplog.handler)
warnings: Final = {
record.getMessage() for record in caplog.records if "counts again without" in record.getMessage()
}
(warning,) = warnings
assert all(option in warning for option in _CLAUDE_COUNT_OPTIONS)
@pytest.mark.asyncio
async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-haiku-5-5",
_CLAUDE_COUNT_OPTIONS,
lambda _: httpx.Response(429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow"}}),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 429
assert len(requests) == 1
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
"model": "bedrock/us.anthropic.claude-count-probe-v1:0",
"aws_access_key_id": "AKIATESTACCESSKEY",
"aws_secret_access_key": "test-secret",
"aws_region_name": "us-east-1",
}
async def _count_on_a_bedrock_claude_deployment(
monkeypatch: pytest.MonkeyPatch,
body: Mapping[str, JsonValue],
runtime: Callable[[httpx.Request], httpx.Response],
mantle: Callable[[httpx.Request], httpx.Response],
) -> tuple[object, tuple[httpx.Request, ...]]:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}])
request: Final = _json_request(
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock as bedrock:
bedrock.route(host=_BEDROCK_RUNTIME_HOST).mock(side_effect=runtime)
bedrock.route(host=_BEDROCK_MANTLE_HOST).mock(side_effect=mantle)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
return response, tuple(call.request for call in bedrock.calls)
def _runtime_counts_only_without_options(request: httpx.Request) -> httpx.Response:
if "invokeModel" in json.loads(request.content)["input"]:
return httpx.Response(400, json={"message": "thinking: unsupported"})
return httpx.Response(200, json={"inputTokens": 9})
def _mantle_unreachable(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("bedrock-mantle is unreachable", request=request)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"malformed",
(
{"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}]},
{"messages": [{"role": "user", "content": 5}]},
{"messages": [{"role": "user", "content": [{"text": "Weather in SF?"}]}]},
),
ids=("tool_type_not_a_string", "content_not_iterable", "block_without_type"),
)
async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_request_again_without_the_options(
monkeypatch: pytest.MonkeyPatch, malformed: dict[str, JsonValue]
) -> None:
response, requests = await _count_on_a_bedrock_claude_deployment(
monkeypatch,
{**malformed, "thinking": {"type": "enabled", "budget_tokens": 2000}},
_runtime_counts_only_without_options,
_mantle_unreachable,
)
assert not isinstance(response, JSONResponse)
assert response["input_tokens"] == 9
assert [request.url.host for request in requests] == [_BEDROCK_RUNTIME_HOST]
assert "invokeModel" not in json.loads(requests[0].content)["input"]
@pytest.mark.asyncio
async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_options_when_mantle_is_unreachable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response, requests = await _count_on_a_bedrock_claude_deployment(
monkeypatch,
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
_runtime_counts_only_without_options,
_mantle_unreachable,
)
assert not isinstance(response, JSONResponse)
assert response["input_tokens"] == 9
hosts: Final = [request.url.host for request in requests]
assert hosts[0] == hosts[-1] == _BEDROCK_RUNTIME_HOST
assert _BEDROCK_MANTLE_HOST in hosts
assert "invokeModel" not in json.loads(requests[-1].content)["input"]
_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management"
_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management"
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
_CLAUDE_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
class _RoutesTo(CustomRoutingStrategyBase):
def __init__(self, router: Router, api_key: str) -> None:
self._router: Final = router
self._api_key: Final = api_key
async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]:
return next(
deployment
for deployment in self._router.model_list
if deployment["litellm_params"]["api_key"] == self._api_key
)
def _mixed_drop_group(routed_key: str) -> Router:
router: Final = Router(
model_list=[
{
"model_name": "claude-agent",
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": _KEEPS_CONTEXT_MANAGEMENT},
},
{
"model_name": "claude-agent",
"litellm_params": {
"model": "anthropic/claude-haiku-5-5",
"api_key": _DROPS_CONTEXT_MANAGEMENT,
"additional_drop_params": ["context_management"],
},
},
]
)
router.set_custom_routing_strategy(_RoutesTo(router, routed_key))
return router
@pytest.mark.asyncio
@pytest.mark.parametrize("routed_key", (_KEEPS_CONTEXT_MANAGEMENT, _DROPS_CONTEXT_MANAGEMENT))
async def test_anthropic_count_tokens_in_a_mixed_drop_group_sends_context_management_when_the_routed_inference_does(
monkeypatch: pytest.MonkeyPatch, routed_key: str
) -> None:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key))
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
messages: Final = [{"role": "user", "content": "Weather in SF?"}]
with respx.mock:
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
)
count_route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 7})
)
await _mixed_drop_group(routed_key).aanthropic_messages(
model="claude-agent", messages=messages, max_tokens=16, context_management=_CONTEXT_EDITS
)
response: Final = await anthropic_count_tokens(
_json_request(
{"model": "claude-agent", "messages": messages, "context_management": _CONTEXT_EDITS},
"python-httpx/0.28.1",
),
UserAPIKeyAuth(),
)
assert not isinstance(response, JSONResponse)
inference: Final = inference_route.calls.last.request
count: Final = count_route.calls.last.request
assert inference.headers["x-api-key"] == count.headers["x-api-key"] == routed_key
assert ("context_management" in json.loads(count.content)) == (
"context_management" in json.loads(inference.content)
)
assert ("context_management" in json.loads(count.content)) == (routed_key == _KEEPS_CONTEXT_MANAGEMENT)
_AZURE_AI_CLAUDE: Final[dict[str, str]] = {
"model": "azure_ai/claude-haiku-5-5",
"api_base": "https://example-resource.services.ai.azure.com/anthropic",
"api_key": "azure-test-key",
}
_ANTHROPIC_CLAUDE: Final[dict[str, str]] = {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-ant-test"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("litellm_params", "general_settings", "forwarded_model_groups", "caller_beta_sent"),
(
(_ANTHROPIC_CLAUDE, {}, None, True),
(_AZURE_AI_CLAUDE, {}, None, False),
(_AZURE_AI_CLAUDE, {"forward_client_headers_to_llm_api": True}, None, True),
(_AZURE_AI_CLAUDE, {}, ["claude-agent"], True),
),
ids=(
"anthropic-api-header-provider",
"other-provider",
"other-provider-forwarding-client-headers",
"other-provider-in-a-forwarding-model-group",
),
)
async def test_anthropic_count_tokens_sends_the_caller_beta_only_when_inference_does(
monkeypatch: pytest.MonkeyPatch,
litellm_params: dict[str, str],
general_settings: dict[str, object],
forwarded_model_groups: list[str] | None,
caller_beta_sent: bool,
) -> None:
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
from litellm.types.router import ModelGroupSettings
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": litellm_params}])
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(proxy_server, "general_settings", general_settings)
monkeypatch.setattr(
litellm, "model_group_settings", ModelGroupSettings(forward_client_headers_to_llm_api=forwarded_model_groups)
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
body: Final[dict[str, JsonValue]] = {
"model": "claude-agent",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"thinking": {"type": "enabled", "budget_tokens": 2000},
}
caller_headers: Final = {"anthropic-beta": _CALLER_BETA}
proxy_data: Final = await add_litellm_data_to_request(
data=dict(body),
request=_json_request(body, "python-httpx/0.28.1", caller_headers),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"),
proxy_config=proxy_server.ProxyConfig(),
general_settings=general_settings,
)
with respx.mock:
inference_route: Final = respx.post(url__regex=r".*/v1/messages$").mock(
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
)
count_route: Final = respx.post(url__regex=r".*/v1/messages/count_tokens$").mock(
return_value=httpx.Response(200, json={"input_tokens": 7})
)
await router.aanthropic_messages(
**body,
max_tokens=4096,
**{name: proxy_data[name] for name in ("provider_specific_header", "headers") if name in proxy_data},
)
response: Final = await anthropic_count_tokens(
_json_request(body, "python-httpx/0.28.1", caller_headers), UserAPIKeyAuth(api_key="sk-1234")
)
assert not isinstance(response, JSONResponse)
inference_betas: Final = inference_route.calls.last.request.headers.get("anthropic-beta", "").split(",")
count_betas: Final = count_route.calls.last.request.headers.get("anthropic-beta", "").split(",")
assert (_CALLER_BETA in inference_betas) == (_CALLER_BETA in count_betas) == caller_beta_sent
async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
},
)
)
await litellm.anthropic_messages(
model=f"anthropic/{model}",
messages=[{"role": "user", "content": "Weather in SF?"}],
max_tokens=4096,
api_key="sk-ant-test",
**options,
)
return json.loads(route.calls.last.request.content)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_drop_params", "user_agent"),
(
({"drop_params": True}, "python-httpx/0.28.1"),
({}, "claude-cli/2.1.0 (external, cli)"),
),
)
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send(
monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True})
kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking})
assert dropped["thinking"] != kept["thinking"]
response, (count,) = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-opus-4-7",
{"thinking": thinking, **request_drop_params},
lambda _: httpx.Response(200, json={"input_tokens": 7}),
user_agent,
)
assert not isinstance(response, JSONResponse)
counted: Final = json.loads(count.content)
assert {name: counted.get(name) for name in ("thinking", "output_config")} == {
name: dropped.get(name) for name in ("thinking", "output_config")
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_drop_params", "request_params"),
(
(True, {"drop_params": "maybe"}),
(True, {"drop_params": False}),
(True, {"drop_params": None}),
(True, {}),
(None, {"drop_params": " true "}),
),
)
async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_deployments_like_inference(
monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_params: Mapping[str, JsonValue]
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
deployment_params: Final[Mapping[str, JsonValue]] = (
{} if deployment_drop_params is None else {"drop_params": deployment_drop_params}
)
with respx.mock:
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
)
await _claude_router("anthropic/claude-opus-5-5", deployment_params).aanthropic_messages(
model="claude-agent",
messages=[{"role": "user", "content": "Weather in SF?"}],
max_tokens=4096,
thinking=thinking,
**request_params,
)
sent: Final = json.loads(inference_route.calls.last.request.content)
response, (count,) = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-opus-5-5",
{"thinking": thinking, **request_params},
lambda _: httpx.Response(200, json={"input_tokens": 7}),
deployment_params=deployment_params,
)
assert not isinstance(response, JSONResponse)
assert json.loads(count.content).get("thinking") == sent.get("thinking")
@pytest.mark.asyncio
async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format(
monkeypatch: pytest.MonkeyPatch,
) -> None:
output_format: Final[dict[str, JsonValue]] = {
"type": "json_schema",
"schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
}
response, (count,) = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-haiku-5-5",
{"output_format": output_format},
lambda _: httpx.Response(200, json={"input_tokens": 7}),
)
assert not isinstance(response, JSONResponse)
counted: Final = json.loads(count.content)
assert "output_format" not in counted
assert counted["output_config"]["format"] == output_format
class _ScriptedCounter(BaseTokenCounter):
def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None:
self._replies = replies
@ -1299,6 +1974,8 @@ class _ScriptedCounter(BaseTokenCounter):
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: object = None,
request_options: object = None,
anthropic_beta: object = None,
) -> TokenCountResponse | None:
self.calls.append((model_to_use, messages, tools, system))
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
@ -3037,7 +3714,7 @@ class _PartnerCountRequest(BaseModel):
async def _vertex_partner_word_count(
model: str, request_data: JsonValue, litellm_params: JsonValue
model: str, request_data: JsonValue, litellm_params: JsonValue, betas: object = ()
) -> dict[str, JsonValue]:
body: Final = _PartnerCountRequest.model_validate(request_data)
return {"input_tokens": sum(len(message.content.split()) for message in body.messages)}