From f84c244f3e0cfd5d3e7960d5b14baa9a3c3c5e32 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:09:46 -0800 Subject: [PATCH] fix(count_tokens): send inference's betas and tools on the Claude count, and retry a rejected count without the forwarded options The Claude count now builds its anthropic-beta set and tools through the same provider config inference uses, so effort, thinking display, the caller's anthropic-beta header, and Bedrock's tool rewrites match the request inference would send. When the provider still rejects a count that carries the forwarded thinking, tool, output, or context options with a 4xx other than 429, the proxy counts once more without them, which is the request it sent before those options were forwarded. Pass-through Bedrock counts go back to detecting Converse or InvokeModel from the body alone --- .../anthropic/count_tokens/token_counter.py | 7 +- .../anthropic/count_tokens/transformation.py | 60 +++-- .../messages/native/transformation.py | 12 + .../anthropic/count_tokens/token_counter.py | 7 +- .../anthropic_messages/transformation.py | 14 +- litellm/llms/base_llm/base_utils.py | 1 + .../count_tokens/bedrock_token_counter.py | 60 +++-- litellm/llms/bedrock/count_tokens/handler.py | 9 +- .../bedrock/count_tokens/transformation.py | 26 +- .../anthropic_claude_transformation.py | 12 + .../bedrock/messages/mantle_transformation.py | 4 + litellm/llms/gemini/common_utils.py | 1 + .../responses/count_tokens/token_counter.py | 1 + litellm/llms/vertex_ai/common_utils.py | 7 +- .../anthropic/messages/transformation.py | 87 ++++--- .../proxy/anthropic_endpoints/endpoints.py | 3 +- .../common_utils/token_count_settings.py | 4 + litellm/proxy/proxy_server.py | 78 ++++-- ...t_anthropic_count_tokens_transformation.py | 87 +++++-- ...est_bedrock_count_tokens_transformation.py | 25 ++ .../test_bedrock_token_counter.py | 101 +++++++- .../vertex_ai/test_vertex_ai_common_utils.py | 3 +- tests/unit/proxy/test_proxy_token_counter.py | 222 +++++++++++++++++- 23 files changed, 654 insertions(+), 177 deletions(-) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index d54732883b4..7e89639deb1 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -51,6 +51,7 @@ class AnthropicTokenCounter(BaseTokenCounter): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ @@ -96,7 +97,7 @@ class AnthropicTokenCounter(BaseTokenCounter): pass claude_options: Final = claude_count_options( - model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, context_management, deployment + model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, anthropic_beta, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "anthropic_api") @@ -138,7 +139,9 @@ class AnthropicTokenCounter(BaseTokenCounter): else [dict(message) for message in claude_options.messages], auth_header=auth_header, api_base=api_base, - tools=tools, + tools=tools + if claude_options.messages is None + else [dict(tool) for tool in claude_options.tools or ()] or None, system=system, optional_params={**count_options, **claude_options.options}, betas=claude_options.betas, diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 6a9d0aa4d65..3f6d68f1e4c 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -29,7 +29,7 @@ from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes -from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) @@ -337,31 +337,41 @@ class ClaudeCountOptions: options: Mapping[str, JsonValue] betas: tuple[str, ...] messages: tuple[Mapping[str, JsonValue], ...] | None + tools: tuple[Mapping[str, JsonValue], ...] | None -NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None) +NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None) _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) _MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1 +def _with_provider_betas(headers: Mapping[str, str], provider: str) -> Mapping[str, str]: + from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers + + beta_header: Final = headers.get("anthropic-beta") + if beta_header is None: + return headers + supported: Final = filter_and_transform_beta_headers( + [beta.strip() for beta in beta_header.split(",") if beta.strip()], provider + ) + other_headers: Final = {name: value for name, value in headers.items() if name != "anthropic-beta"} + return {**other_headers, "anthropic-beta": ",".join(supported)} if supported else other_headers + + def claude_count_options( model: str, provider: LlmProviders, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, request_options: Mapping[str, JsonValue], - context_management: CountContextManagement | None, + anthropic_beta: str | None, deployment: Mapping[str, object] | None, ) -> ClaudeCountOptions | ChatCountTranslationError: - sent: Final = { - **{name: value for name, value in request_options.items() if value is not None}, - **({} if context_management is None else {"context_management": context_management.spec}), - } + sent: Final = {name: value for name, value in request_options.items() if value is not None} if not sent: return NO_CLAUDE_COUNT_OPTIONS - from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.exceptions import BadRequestError - from litellm.llms.anthropic.common_utils import AnthropicError, context_management_betas + from litellm.llms.anthropic.common_utils import AnthropicError from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager @@ -372,37 +382,39 @@ def claude_count_options( deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) leading_system, conversation = split_leading_system_run(messages) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) - sent_tools: Final = copy.deepcopy([dict(tool) for tool in tools or ()]) + sent_params: Final = { + **copy.deepcopy(sent), + **({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}), + } + beta_headers: Final = messages_config.headers_with_anthropic_betas( + {"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation + ) + sent_headers: Final = dict( + _with_provider_betas(beta_headers, provider.value) + if messages_config.should_filter_anthropic_beta_headers() + else beta_headers + ) try: request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts model=model, messages=sent_conversation, - anthropic_messages_optional_request_params={ - **copy.deepcopy(sent), - **({"tools": sent_tools} if sent_tools else {}), - "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET, - }, + anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET}, litellm_params=GenericLiteLLMParams.model_validate( { "drop_params": deployment_params.get("drop_params"), "model_info": resolved_deployment.get("model_info"), } ), - headers={}, + headers=sent_headers, ) except (AnthropicError, BadRequestError) as e: return ChatCountTranslationError(message=str(e.message)) transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) - options: Final = _count_options(transformed) - betas: Final = sorted(context_management_betas(options.get("context_management"))) return ClaudeCountOptions( - options=options, - betas=tuple( - filter_and_transform_beta_headers(betas, provider.value) - if messages_config.should_filter_anthropic_beta_headers() - else betas - ), + options=_count_options(transformed), + betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)), messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]), + tools=transformed.get("tools"), ) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 54edc273a9e..152ca2628cc 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -413,6 +413,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages=messages, ) + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + return self._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared beta step takes and returns bare dicts + headers=dict(headers), + optional_params=dict(optional_params), + messages=messages, + ) + @staticmethod def _translate_reasoning_effort_to_anthropic( model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 4b2fc4200ca..7ca2c813c76 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -65,6 +65,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ @@ -110,7 +111,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): pass claude_options: Final = claude_count_options( - model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, context_management, deployment + model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, anthropic_beta, deployment ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api") @@ -142,7 +143,9 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): api_key=api_key, api_base=api_base, litellm_params=litellm_params, - tools=tools, + tools=tools + if claude_options.messages is None + else [dict(tool) for tool in claude_options.tools or ()] or None, system=system, optional_params={**count_options, **claude_options.options}, betas=claude_options.betas, diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bda7015ab2b..febb1d86244 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx @@ -127,6 +127,18 @@ class BaseAnthropicMessagesConfig(ABC): """ return headers, None + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + return headers + + def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: + beta_header: Final = next((value for name, value in headers.items() if name.lower() == "anthropic-beta"), "") + return tuple(sorted(frozenset(beta.strip() for beta in beta_header.split(",")) - {""})) + def should_filter_anthropic_beta_headers(self) -> bool: """ Whether ``anthropic-beta`` header values should be filtered down to the diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index fc84a5aab30..af1f3b90f08 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -43,6 +43,7 @@ class BaseTokenCounter(ABC): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: pass diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 440354dd01e..b1b95e9079b 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -2,7 +2,7 @@ Bedrock Token Counter implementation using the CountTokens API. """ -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Any, Final @@ -117,21 +117,19 @@ class BedrockTokenCounter(BaseTokenCounter): ) -> bool: return custom_llm_provider == LlmProviders.BEDROCK.value - async def _count_with( - self, - handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler, - tokenizer_type: str, - request_data: Mapping[str, object], - litellm_params: dict[str, object], - resolved_model: str, - ) -> CountTokensOutcome: + def _count_on_mantle( + self, request_data: Mapping[str, object], litellm_params: Mapping[str, object], resolved_model: str + ) -> Awaitable[Mapping[str, JsonValue]]: + return self._mantle_handler.handle_count_tokens_request( + request_data=dict(request_data), + litellm_params=dict(litellm_params), + resolved_model=resolved_model, + client=self._client, + ) + + async def _count_with(self, count: Awaitable[Mapping[str, JsonValue]], tokenizer_type: str) -> CountTokensOutcome: try: - result: Final = await handler.handle_count_tokens_request( - request_data=dict(request_data), - litellm_params=litellm_params, - resolved_model=resolved_model, - client=self._client, - ) + result: Final = await count reply: Final = _CountTokensReply.model_validate(result) except BedrockError as e: verbose_logger.debug( @@ -157,6 +155,7 @@ class BedrockTokenCounter(BaseTokenCounter): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: if not messages: return None @@ -182,14 +181,15 @@ class BedrockTokenCounter(BaseTokenCounter): if isinstance(translated, ChatCountTranslationError): return translated.response(request_model, model_to_use, primary_tokenizer) sent_messages: Final = messages if translated is None else translated.messages + sent_tools: Final = tools if translated is None else translated.tools claude_options: Final = ( claude_count_options( counted_model, LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, sent_messages, - tools if translated is None else translated.tools, + sent_tools, request_options, - context_management, + anthropic_beta, deployment, ) if is_mantle_claude_model(resolved_model) @@ -197,29 +197,37 @@ class BedrockTokenCounter(BaseTokenCounter): ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, primary_tokenizer) - counted_messages: Final = sent_messages if claude_options.messages is None else claude_options.messages + invoke_fields: Final = _invoke_fields(claude_options) runtime_request: Final = { - **( - _request_data(model_to_use, counted_messages, tools, system) - if translated is None - else _request_data(model_to_use, counted_messages, translated.tools, translated.system) + **_request_data( + model_to_use, + sent_messages if claude_options.messages is None else claude_options.messages, + sent_tools if claude_options.messages is None else claude_options.tools, + system if translated is None else translated.system, ), - **_invoke_fields(claude_options), + **invoke_fields, } mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options} primary: Final = ( await self._count_with( - self._mantle_handler, primary_tokenizer, mantle_request, litellm_params, resolved_model + self._count_on_mantle(mantle_request, litellm_params, resolved_model), primary_tokenizer ) if use_mantle else await self._count_with( - self._runtime_handler, primary_tokenizer, runtime_request, litellm_params, resolved_model + self._runtime_handler.handle_count_tokens_request( + request_data=dict(runtime_request), + litellm_params=litellm_params, + resolved_model=resolved_model, + client=self._client, + input_type="invokeModel" if invoke_fields else None, + ), + primary_tokenizer, ) ) outcome: Final = ( await self._count_with( - self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model + self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE ) if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) else primary diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 5f8f478e4cd..0e64eb46a02 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -12,7 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.llms.bedrock.base_aws_llm import run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig +from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig, BedrockCountTokensInputType from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client @@ -29,6 +29,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): litellm_params: dict[str, Any], resolved_model: str, client: AsyncHTTPHandler | None = None, + input_type: BedrockCountTokensInputType | None = None, ) -> dict[str, Any]: """ Handle a CountTokens request using existing LiteLLM patterns. @@ -43,7 +44,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): """ try: # Validate the request - self.validate_count_tokens_request(request_data) + self.validate_count_tokens_request(request_data, input_type) verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model) @@ -58,7 +59,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): verbose_logger.debug("Retrieved AWS region: %s", aws_region_name) # Transform request to Bedrock format (supports both Converse and InvokeModel) - bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data) + bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens( + request_data=request_data, input_type=input_type + ) verbose_logger.debug("Transformed request: %s", bedrock_request) diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index 00c7f6093ea..5ebf492dde4 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -7,7 +7,7 @@ to AWS Bedrock's CountTokens API format and vice versa. import re from collections.abc import Mapping -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from pydantic import JsonValue @@ -19,9 +19,7 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model # max_tokens field; CountTokens only counts input, so it has no effect # on any generation. DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024 -_INVOKE_MODEL_ONLY_FIELDS: Final = frozenset( - ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta") -) +BedrockCountTokensInputType: TypeAlias = Literal["converse", "invokeModel"] def _invoke_model_max_tokens(thinking: JsonValue) -> int: @@ -86,7 +84,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): - Response: {"inputTokens": } """ - def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> Literal["converse", "invokeModel"]: + def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> BedrockCountTokensInputType: """ Detect whether to use 'converse' or 'invokeModel' input format. @@ -97,8 +95,6 @@ class BedrockCountTokensConfig(BaseAWSLLM): 'converse' or 'invokeModel' """ messages: Final = request_data.get("messages") - if not _INVOKE_MODEL_ONLY_FIELDS.isdisjoint(request_data): - return "invokeModel" if isinstance(messages, list): # Anthropic content blocks carry a "type" key ({"type": "text", ...}); # Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse @@ -119,6 +115,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): def transform_anthropic_to_bedrock_count_tokens( self, request_data: Mapping[str, JsonValue], + input_type: BedrockCountTokensInputType | None = None, ) -> dict[str, JsonValue]: """ Transform request to Bedrock CountTokens format. @@ -149,9 +146,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): } } """ - input_type: Final = self._detect_input_type(request_data) - - if input_type == "converse": + if (input_type or self._detect_input_type(request_data)) == "converse": return self._transform_to_converse_format(request_data) else: return self._transform_to_invoke_model_format(request_data) @@ -270,23 +265,20 @@ class BedrockCountTokensConfig(BaseAWSLLM): return {"input_tokens": input_tokens} - def validate_count_tokens_request(self, request_data: Mapping[str, JsonValue]) -> None: + def validate_count_tokens_request( + self, request_data: Mapping[str, JsonValue], input_type: BedrockCountTokensInputType | None = None + ) -> None: """ Validate the incoming count tokens request. Supports both Converse and InvokeModel input formats. - Args: - request_data: The request payload - Raises: ValueError: If the request is invalid """ if not request_data.get("model"): raise ValueError("model parameter is required") - input_type: Final = self._detect_input_type(request_data) - - if input_type == "converse": + if (input_type or self._detect_input_type(request_data)) == "converse": # Validate Converse format (messages-based) messages: Final = request_data.get("messages", []) if not messages: diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index e1e15861a90..dcb68b3e817 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -76,6 +76,7 @@ else: _MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) class AmazonAnthropicClaudeMessagesConfig( @@ -105,6 +106,17 @@ class AmazonAnthropicClaudeMessagesConfig( def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return True + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + return headers + + def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: + return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ()) + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) def get_error_class( diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 89f4cc4dd8b..9f96e80646f 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -18,6 +18,7 @@ from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, @@ -70,6 +71,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return False + def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: + return BaseAnthropicMessagesConfig.sent_anthropic_betas(self, headers, request) + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 632eaa21292..138afa4051d 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -535,6 +535,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: import copy diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 8953bc6b88d..70f3eb6f097 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -149,6 +149,7 @@ class OpenAITokenCounter(BaseTokenCounter): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: """ Count tokens using OpenAI's Responses API /input_tokens endpoint. diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 7f53346ffde..bd475e54f15 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1453,6 +1453,7 @@ class VertexAITokenCounter(BaseTokenCounter): context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: import copy @@ -1519,7 +1520,7 @@ class VertexAITokenCounter(BaseTokenCounter): sent_messages, tools, request_options, - context_management, + anthropic_beta, deployment, ) if isinstance(claude_options, ChatCountTranslationError): @@ -1556,7 +1557,9 @@ class VertexAITokenCounter(BaseTokenCounter): vertex_location=vertex_location, vertex_credentials=vertex_credentials, system=system, - tools=tools, + tools=tools + if claude_options.messages is None + else [dict(tool) for tool in claude_options.tools or ()] or None, count_options={**count_options, **claude_options.options}, betas=claude_options.betas, ) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 410d532566b..902692dcee4 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence from typing import Final from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, @@ -51,6 +51,11 @@ def _vertex_version(headers: Mapping[str, object], model: str, drop_params: bool ) +def _is_web_search(tool: Mapping[str, object]) -> bool: + tool_type: Final = tool.get("type") + return isinstance(tool_type, str) and tool_type.startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value) + + def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY} @@ -94,10 +99,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert project_id=vertex_ai_project, custom_llm_provider="vertex_ai", ) - return self._finalize_vertex_messages_environment( + vertex_headers, vertex_api_base = self._finalize_vertex_messages_environment( headers, model, - messages, optional_params, api_base, access_token, @@ -105,6 +109,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert vertex_ai_project, vertex_ai_location, ) + return dict(self.headers_with_anthropic_betas(vertex_headers, optional_params, messages)), vertex_api_base async def avalidate_anthropic_messages_environment( self, @@ -122,10 +127,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert project_id=vertex_ai_project, custom_llm_provider="vertex_ai", ) - return self._finalize_vertex_messages_environment( + vertex_headers, vertex_api_base = self._finalize_vertex_messages_environment( headers, model, - messages, optional_params, api_base, access_token, @@ -133,12 +137,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert vertex_ai_project, vertex_ai_location, ) + return dict(self.headers_with_anthropic_betas(vertex_headers, optional_params, messages)), vertex_api_base def _finalize_vertex_messages_environment( self, headers: Mapping[str, str], model: str, - messages: Sequence[object], optional_params: dict, # mutable-ok: preserves the existing provider optional-parameter contract api_base: str | None, access_token: str, @@ -162,39 +166,48 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert request_headers["content-type"] = "application/json" - # Add beta headers for Vertex AI - tools: Final = optional_params.get("tools", []) - beta_values: Final[set[str]] = set() - - # Get existing beta headers if any - existing_beta: Final = request_headers.get("anthropic-beta") - if existing_beta: - beta_values.update(b.strip() for b in existing_beta.split(",")) - - beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict - - # Check for web search tool - for tool in tools: - if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value): - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value) - break - - # Check for tool search tools - Vertex AI uses different beta header - anthropic_model_info: Final = AnthropicModelInfo() - if anthropic_model_info.is_tool_search_used(tools): - beta_values.add(get_tool_search_beta_header("vertex_ai")) - - if optional_params.get("safeguards") is not None: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value) - - if messages_carry_output_config(messages): - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) - - if beta_values: - request_headers["anthropic-beta"] = ",".join(beta_values) - return request_headers, api_base + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + tools: Final = tuple( + tool for tool in map(as_str_mapping, iter_items(optional_params.get("tools") or ())) if tool is not None + ) + existing_betas: Final = (headers.get("anthropic-beta") or "").split(",") + beta_values: Final = frozenset( + ( + *(beta.strip() for beta in existing_betas if beta.strip()), + *context_management_betas(optional_params.get("context_management")), # pyright: ignore[reportArgumentType] # optional_params is the config contract's untyped request body + *( + (ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value,) + if any(map(_is_web_search, tools)) + else () + ), + *( + (get_tool_search_beta_header("vertex_ai"),) + if AnthropicModelInfo().is_tool_search_used(list(tools)) + else () + ), + *( + (ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,) + if optional_params.get("safeguards") is not None + else () + ), + *( + (ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value,) + if messages_carry_output_config(messages) + else () + ), + ) + ) + if not beta_values: + return headers + return {**headers, "anthropic-beta": ",".join(sorted(beta_values))} + def get_complete_url( self, api_base: str | None, diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index f5c6f6f48dc..9015cee47d5 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -336,7 +336,7 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue: _COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params") -_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format") +_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format", "context_management") class _CountTokensBodyOptions(TypedDict): @@ -429,6 +429,7 @@ async def count_tokens( messages_format="anthropic", litellm_call_id=litellm_call_id, client_oauth_token=forwarded_anthropic_oauth_token(request.headers), + anthropic_beta=request.headers.get("anthropic-beta"), **_count_tokens_body_options(data, auto_drop_params), ) except HTTPException as e: diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index dd04e531f2d..5763f0b2067 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -72,6 +72,10 @@ class ProviderCountFailure: def denied_in_region(self) -> bool: return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES + @property + def rejected_request(self) -> bool: + return 400 <= self.status_code < 500 and self.status_code != 429 + def with_hint(self, hint: str) -> "ProviderCountFailure": if self.rejected_before_provider_call: return self diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e228266ba4d..7db5b6e5a29 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14813,6 +14813,7 @@ async def _try_provider_token_count( context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> Union["TokenCountResponse", ProviderCountFailure, None]: """Attempt provider-specific token counting. Returns None to fall through to local counting.""" if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): @@ -14841,6 +14842,7 @@ async def _try_provider_token_count( context_management=context_management, chat_params=chat_params, request_options=request_options, + anthropic_beta=anthropic_beta, ) except httpx.HTTPStatusError as e: return ProviderCountFailure( @@ -14923,6 +14925,32 @@ async def _provider_token_count( return failure +def _retried_without_rejected_options( + count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]", + request_options: Mapping[str, JsonValue], +) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]: + if not request_options: + return count_with + return partial(_count_retrying_without_options, count_with) + + +async def _count_retrying_without_options( + count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]", + model: str, + region: str | None, +) -> Union["TokenCountResponse", ProviderCountFailure, None]: + counted: Final = await count_with(model, region) + if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): + return counted + verbose_proxy_logger.warning( + "Token count for %s answered %s with the request's thinking, tool, and output options, " + "so it counts again without them", + model, + counted.status_code, + ) + return await count_with(model, region, request_options=MappingProxyType({})) + + @dataclass(frozen=True, slots=True) class _LocalTokenEstimate: total_tokens: int @@ -15335,6 +15363,7 @@ async def count_request_tokens( litellm_call_id: str | None = None, client_oauth_token: str | None = None, request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse: call_id: Final = resolve_litellm_call_id(litellm_call_id) if request.prompt is None and request.messages is None and request.contents is None: @@ -15355,6 +15384,7 @@ async def count_request_tokens( responses_request=responses_request, client_oauth_token=client_oauth_token, request_options=request_options, + anthropic_beta=anthropic_beta, ) return await logged_token_count( _with_anthropic_content_errors(count) if messages_format == "anthropic" else count, @@ -15398,6 +15428,7 @@ async def _count_request_tokens( responses_request: Mapping[str, object] | None, client_oauth_token: str | None, request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15474,6 +15505,7 @@ async def _count_request_tokens( ) if isinstance(chat_count, ChatCountBridgeError): raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400) + counted_options: Final = _count_request_options(request_options, count_deployment, sent_params) ladder: Final = ( None if provider_counter is None @@ -15485,27 +15517,33 @@ async def _count_request_tokens( routed_deployment, request.model, ) - or partial( - _try_provider_token_count, - provider_counter, - custom_llm_provider, - messages=( - [dict(message) for message in chat_count.messages] - if chat_count is not None - else [{"role": "user", "content": prompt}] - if prompt_as_message - else request_messages + or _retried_without_rejected_options( + partial( + _try_provider_token_count, + provider_counter, + custom_llm_provider, + messages=( + [dict(message) for message in chat_count.messages] + if chat_count is not None + else [{"role": "user", "content": prompt}] + if prompt_as_message + else request_messages + ), + contents=None if prompt_as_message else contents, + deployment=count_deployment, + request_model=request.model, + tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None, + system=system, + messages_format=messages_format, + output_format=output_format, + context_management=_count_context_management( + context_management, request.model, request_litellm_params + ), + chat_params=MappingProxyType({}) if chat_count is None else chat_count.params, + request_options=counted_options, + anthropic_beta=anthropic_beta, ), - contents=None if prompt_as_message else contents, - deployment=count_deployment, - request_model=request.model, - tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None, - system=system, - messages_format=messages_format, - output_format=output_format, - context_management=_count_context_management(context_management, request.model, request_litellm_params), - chat_params=MappingProxyType({}) if chat_count is None else chat_count.params, - request_options=_count_request_options(request_options, count_deployment, sent_params), + counted_options, ), model_to_use=model_to_use, request_model=request.model, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 9e818fae5c4..4391617295e 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -27,8 +27,12 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) -from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES -from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse +from litellm.types.llms.anthropic import ( + ANTHROPIC_ADVISOR_TOOL_TYPE, + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) +from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @pytest.mark.parametrize( @@ -488,7 +492,7 @@ async def _counted_requests( messages_format: TokenCountMessagesFormat, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), request_options: Mapping[str, JsonValue] = MappingProxyType({}), - context_management: CountContextManagement | None = None, + anthropic_beta: str | None = None, model: str = "claude-opus-5-5", ) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -509,7 +513,7 @@ async def _counted_requests( messages_format=messages_format, chat_params=chat_params, request_options=request_options, - context_management=context_management, + anthropic_beta=anthropic_beta, ) return result, tuple(sent.get() for _ in range(sent.qsize())) @@ -880,12 +884,6 @@ _MESSAGE_REPLY: Final[dict[str, JsonValue]] = { _INFERENCE_PROVIDERS: Final = MappingProxyType( {AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"} ) -_CONTEXT_BETAS: Final = frozenset( - ( - ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, - ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, - ) -) _WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA} _ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000} @@ -894,6 +892,8 @@ async def _messages_inference_request( counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], inference_endpoint: str, request_options: Mapping[str, JsonValue], + model: str = "claude-haiku-5-5", + extra_headers: Mapping[str, str] = MappingProxyType({}), ) -> httpx.Request: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -904,12 +904,13 @@ async def _messages_inference_request( with respx.mock: respx.post(inference_endpoint).mock(side_effect=reply) await litellm.anthropic_messages( - model=f"{_INFERENCE_PROVIDERS[counter_type]}/claude-haiku-5-5", + model=f"{_INFERENCE_PROVIDERS[counter_type]}/{model}", messages=[{"role": "user", "content": "Weather in SF?"}], tools=[_WEATHER_TOOL], max_tokens=4096, api_key="sk-ant-api03-test-key", api_base="https://gateway.example", + extra_headers=dict(extra_headers), **request_options, ) return sent.get() @@ -923,16 +924,24 @@ def _betas(request: httpx.Request) -> frozenset[str]: @pytest.mark.usefixtures("httpx_transport_clients") @_COUNTERS @pytest.mark.parametrize( - ("request_options", "context_management"), + ("request_options", "expected_betas"), ( - ({"thinking": _ENABLED_THINKING}, None), - ({"tool_choice": {"type": "any"}}, None), - ({"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, None), + ({"thinking": _ENABLED_THINKING}, frozenset()), + ({"tool_choice": {"type": "any"}}, frozenset()), ( - {"thinking": _ENABLED_THINKING}, - {"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]}, + {"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, + frozenset((ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value,)), ), - ({}, {"edits": [{"type": "compact_20260112"}]}), + ( + { + "thinking": _ENABLED_THINKING, + "context_management": { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] + }, + }, + frozenset((ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,)), + ), + ({"context_management": {"edits": [{"type": "compact_20260112"}]}}, frozenset()), ), ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"), ) @@ -940,12 +949,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str, request_options: dict[str, JsonValue], - context_management: dict[str, JsonValue] | None, + expected_betas: frozenset[str], ) -> None: inference: Final = await _messages_inference_request( - counter_type, - count_endpoint.removesuffix("/count_tokens"), - {**request_options, **({} if context_management is None else {"context_management": context_management})}, + counter_type, count_endpoint.removesuffix("/count_tokens"), request_options ) _, (count,) = await _counted_requests( counter_type, @@ -954,9 +961,6 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se [_WEATHER_TOOL], "anthropic", request_options=request_options, - context_management=( - None if context_management is None else CountContextManagement(spec=context_management, drop_invalid_edits=False) - ), model="claude-haiku-5-5", ) @@ -973,7 +977,38 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") } - assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS + assert expected_betas <= _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_sends_the_thinking_display_and_caller_betas_messages_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + caller_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value + display_thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}} + inference: Final = await _messages_inference_request( + counter_type, + count_endpoint.removesuffix("/count_tokens"), + display_thinking, + model="claude-opus-4-6", + extra_headers={"anthropic-beta": caller_beta}, + ) + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + request_options=display_thinking, + anthropic_beta=caller_beta, + model="claude-opus-4-6", + ) + + counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} + assert counted_betas == _betas(inference) + assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas @pytest.mark.asyncio diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py index 052baf601f9..7b2b9138093 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py @@ -1,5 +1,8 @@ import base64 import json +from typing import Final + +import pytest from litellm.llms.bedrock.count_tokens.transformation import ( DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, @@ -53,6 +56,28 @@ def test_detect_input_type_converse_blocks_route_to_converse(): assert config._detect_input_type(request) == "converse" +@pytest.mark.parametrize( + "option", + ( + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + {"tool_choice": {"type": "auto"}}, + {"output_config": {"effort": "low"}}, + ), +) +def test_pass_through_body_with_string_content_is_counted_as_converse_whatever_options_it_carries( + option: dict[str, object], +) -> None: + request: Final = { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "messages": [{"role": "user", "content": "hi"}], + **option, + } + + result: Final = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request) + + assert result["input"] == {"converse": {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}} + + def test_transform_to_invoke_model_format_base64_encodes_body(): """The CountTokens API expects invokeModel.body as a base64-encoded blob; Anthropic Messages bodies additionally need anthropic_version/max_tokens diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index d3534dd39b9..425953c6f51 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -10,7 +10,10 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.types.utils import CountContextManagement +from litellm.types.llms.anthropic import ( + ANTHROPIC_EFFORT_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" @@ -350,29 +353,36 @@ _THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = { _WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}} -async def _messages_inference_and_count(deployment_model: str, model_to_use: str) -> tuple[_Bedrock, _Bedrock]: +_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS} + + +async def _messages_inference_and_count( + deployment_model: str, + model_to_use: str, + request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS, + tool: Mapping[str, JsonValue] = _WEATHER_TOOL, + runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}), +) -> tuple[_Bedrock, _Bedrock]: credentials: Final = DEPLOYMENT["litellm_params"] inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) await litellm.anthropic_messages( model=deployment_model, messages=MESSAGES, - tools=[_WEATHER_TOOL], + tools=[dict(tool)], max_tokens=4096, - context_management=_CONTEXT_EDITS, client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), - **_THINKING_AND_TOOL_CHOICE, + **request_options, **credentials, ) - bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177})) result: Final = await _counter(bedrock).count_tokens( model_to_use=model_to_use, messages=MESSAGES, contents=None, deployment={"litellm_params": {**credentials, "model": deployment_model}}, request_model="claude", - tools=[_WEATHER_TOOL], - request_options=_THINKING_AND_TOOL_CHOICE, - context_management=CountContextManagement(spec=_CONTEXT_EDITS, drop_invalid_edits=False), + tools=[dict(tool)], + request_options=request_options, ) assert result is not None assert result.error is False @@ -419,6 +429,79 @@ async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_infe assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"] +_DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}} + + +def _sent_betas(request: httpx.Request) -> tuple[str, ...]: + return tuple(sorted(request.headers["anthropic-beta"].split(","))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "request_options", "expected_beta"), + ( + ( + "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", + {"output_config": {"effort": "low"}}, + ANTHROPIC_EFFORT_BETA_HEADER, + ), + ("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER), + ), + ids=("effort", "thinking_display"), +) +async def test_runtime_count_sends_the_betas_invoke_inference_sends( + deployment_model: str, request_options: dict[str, JsonValue], expected_beta: str +) -> None: + inference, bedrock = await _messages_inference_and_count( + deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta") + assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta") + assert isinstance(counted_betas, list) and expected_beta in counted_betas + + +@pytest.mark.asyncio +async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/invoke/us.anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + _DISPLAY_THINKING, + runtime=(400, UNSUPPORTED), + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) + sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta") + assert isinstance(sent_betas, list) + assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in sent_betas + + +@pytest.mark.asyncio +async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + assert _sent_betas(bedrock.requests[0]) == _sent_betas(inference.requests[0]) + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0]) + + +@pytest.mark.asyncio +async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/invoke/us.anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + _THINKING_AND_TOOL_CHOICE, + tool={**_WEATHER_TOOL, "eager_input_streaming": True}, + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] + + _CITY_SCHEMA: Final[dict[str, JsonValue]] = { "type": "object", "properties": {"city": {"type": "string"}}, diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 4a9eedb7dbb..782074a344c 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -2016,8 +2016,7 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert contents=None, deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, tools=tools, - request_options=options, - context_management=CountContextManagement(spec=context_edits, drop_invalid_edits=False), + request_options={**options, "context_management": context_edits}, ) assert result is not None diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 578343165f7..cbae1aac1fb 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -56,6 +56,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( kimi_k3_tokens, load_kimi_k3_encoding, ) +from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensInputType from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http @@ -368,6 +369,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): litellm_call_id=None, client_oauth_token=None, request_options=None, + anthropic_beta=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -451,6 +453,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): litellm_call_id=None, client_oauth_token=None, request_options=None, + anthropic_beta=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -940,6 +943,7 @@ class _FailingRuntimeHandler(BedrockCountTokensHandler): litellm_params: dict[str, object], resolved_model: str, client: AsyncHTTPHandler | None = None, + input_type: BedrockCountTokensInputType | None = None, ) -> dict[str, JsonValue]: raise self._error @@ -1272,6 +1276,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): litellm_call_id=None, client_oauth_token=None, request_options=None, + anthropic_beta=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1332,6 +1337,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): litellm_call_id=None, client_oauth_token=None, request_options=None, + anthropic_beta=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1392,6 +1398,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): litellm_call_id=None, client_oauth_token=None, request_options=None, + anthropic_beta=None, ): raise ProxyException( message="Rate limit exceeded", @@ -1416,7 +1423,9 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = original_token_counter -def _json_request(body: Mapping[str, object], user_agent: str) -> Request: +def _json_request( + body: Mapping[str, object], user_agent: str, headers: Mapping[str, str] = MappingProxyType({}) +) -> Request: payload: Final = json.dumps(body).encode() async def receive() -> Message: @@ -1428,7 +1437,11 @@ def _json_request(body: Mapping[str, object], user_agent: str) -> Request: "method": "POST", "path": "/v1/messages/count_tokens", "query_string": b"", - "headers": [(b"content-type", b"application/json"), (b"user-agent", user_agent.encode())], + "headers": [ + (b"content-type", b"application/json"), + (b"user-agent", user_agent.encode()), + *((name.encode(), value.encode()) for name, value in headers.items()), + ], }, receive, ) @@ -1641,10 +1654,11 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]: @pytest.mark.asyncio @pytest.mark.parametrize( - ("additional_drop_params", "expected_paths"), + ("additional_drop_params", "request_params", "expected_paths"), ( ( None, + {}, { "thinking", "tool_choice", @@ -1656,17 +1670,32 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]: ), ( ["tool_choice", "output_config.format"], + {}, {"thinking", "output_config", "output_config.effort", "context_management"}, ), ( ["context_management"], + {}, {"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"}, ), + ( + None, + {"additional_drop_params": ["thinking", "context_management"]}, + {"tool_choice", "output_config", "output_config.effort", "output_config.format"}, + ), + ), + ids=( + "nothing-dropped", + "deployment-drops-tool-choice-and-the-format", + "deployment-drops-context-management", + "request-drops-thinking-and-context-management", ), - ids=("nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management"), ) async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends( - monkeypatch: pytest.MonkeyPatch, additional_drop_params: list[str] | None, expected_paths: set[str] + monkeypatch: pytest.MonkeyPatch, + additional_drop_params: list[str] | None, + request_params: dict[str, JsonValue], + expected_paths: set[str], ) -> None: import litellm.proxy.proxy_server as proxy_server @@ -1688,6 +1717,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i "messages": [{"role": "user", "content": "Weather in SF?"}], "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], **_CLAUDE_COUNT_OPTIONS, + **request_params, }, "python-httpx/0.28.1", ) @@ -1709,6 +1739,187 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i ) +_CALLER_BETA: Final = "web-fetch-2025-09-10" + + +async def _count_on_a_claude_deployment( + monkeypatch: pytest.MonkeyPatch, + deployment_model: str, + body: Mapping[str, JsonValue], + reply: Callable[[httpx.Request], httpx.Response], + user_agent: str = "python-httpx/0.28.1", +) -> tuple[object, tuple[httpx.Request, ...]]: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + {"model_name": "claude-agent", "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test"}} + ] + ) + request: Final = _json_request( + {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, + user_agent, + {"anthropic-beta": _CALLER_BETA}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(side_effect=reply) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + return response, tuple(call.request for call in route.calls) + + +def _counts_only_without_options(request: httpx.Request) -> httpx.Response: + if _sent_option_paths(json.loads(request.content)): + return httpx.Response( + 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}} + ) + return httpx.Response(200, json={"input_tokens": 7}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "options", "rejected_by_anthropic"), + ( + ("anthropic/claude-haiku-4-5", {"output_config": {"effort": "fastest"}}, False), + ("anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, True), + ), + ids=("rejected-by-the-count-transform", "rejected-by-anthropic"), +) +async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_claude_count_rejects_them( + monkeypatch: pytest.MonkeyPatch, + deployment_model: str, + options: dict[str, JsonValue], + rejected_by_anthropic: bool, +) -> None: + response, requests = await _count_on_a_claude_deployment( + monkeypatch, deployment_model, options, _counts_only_without_options + ) + + assert not isinstance(response, JSONResponse) + assert response["input_tokens"] == 7 + assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [ + *((True,) if rejected_by_anthropic else ()), + False, + ] + assert [_CALLER_BETA in sent.headers.get("anthropic-beta", "") for sent in requests] == [ + *((True,) if rejected_by_anthropic else ()), + False, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", (401, 403, 404)) +async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_options_also_gets( + monkeypatch: pytest.MonkeyPatch, status_code: int +) -> None: + response, requests = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-haiku-5-5", + _CLAUDE_COUNT_OPTIONS, + lambda _: httpx.Response(status_code, json={"type": "error", "error": {"type": "error", "message": "denied"}}), + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == status_code + assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [True, False] + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, requests = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-haiku-5-5", + _CLAUDE_COUNT_OPTIONS, + lambda _: httpx.Response(429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow"}}), + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 429 + assert len(requests) == 1 + + +async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + ) + ) + await litellm.anthropic_messages( + model=f"anthropic/{model}", + messages=[{"role": "user", "content": "Weather in SF?"}], + max_tokens=4096, + api_key="sk-ant-test", + **options, + ) + return json.loads(route.calls.last.request.content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_drop_params", "user_agent"), + (({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")), +) +async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send( + monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"} + dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True}) + kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking}) + assert dropped["thinking"] != kept["thinking"] + + response, (count,) = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-opus-4-7", + {"thinking": thinking, **request_drop_params}, + lambda _: httpx.Response(200, json={"input_tokens": 7}), + user_agent, + ) + + assert not isinstance(response, JSONResponse) + counted: Final = json.loads(count.content) + assert {name: counted.get(name) for name in ("thinking", "output_config")} == { + name: dropped.get(name) for name in ("thinking", "output_config") + } + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + output_format: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + + response, (count,) = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-haiku-5-5", + {"output_format": output_format}, + lambda _: httpx.Response(200, json={"input_tokens": 7}), + ) + + assert not isinstance(response, JSONResponse) + counted: Final = json.loads(count.content) + assert "output_format" not in counted + assert counted["output_config"]["format"] == output_format + + class _ScriptedCounter(BaseTokenCounter): def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: self._replies = replies @@ -1734,6 +1945,7 @@ class _ScriptedCounter(BaseTokenCounter): context_management: object = None, chat_params: object = None, request_options: object = None, + anthropic_beta: object = None, ) -> TokenCountResponse | None: self.calls.append((model_to_use, messages, tools, system)) self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))