diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..29e4121c1c0 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -12,7 +12,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar from urllib.parse import quote, urlparse import httpx -from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError +from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -45,6 +45,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -347,6 +348,20 @@ def _beta_header_values(side: str | Sequence[str] | None) -> tuple[str, ...]: return tuple(entry for entry in side if isinstance(entry, str)) +def context_management_betas(context_management: JsonValue) -> frozenset[str]: + """The anthropic-beta values a ``context_management`` body needs: the compact beta for a + ``compact_20260112`` edit and the context-management beta for every other edit.""" + edits: Final = context_management.get("edits") if isinstance(context_management, dict) else None + if not isinstance(edits, list): + return frozenset() + return frozenset( + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value + if isinstance(edit, dict) and edit.get("type") == "compact_20260112" + else ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + for edit in edits + ) + + def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta: str | Sequence[str] | None) -> str: """Merge anthropic-beta header values, deduplicated and sorted. @@ -1306,8 +1321,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index a230294ad3f..c40ae4c86b4 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -4,7 +4,7 @@ Anthropic CountTokens API handler. Uses httpx for HTTP requests instead of the Anthropic SDK. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final import httpx @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + betas: Sequence[str] = (), ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -79,7 +80,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers - headers: Final = self.get_count_tokens_headers(auth_header) + headers: Final = self.get_count_tokens_headers(auth_header, betas) # Use LiteLLM's async httpx client async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..0498b00e1e6 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -16,6 +16,7 @@ from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, ChatCountTranslationError, anthropic_count_fields, + claude_count_options, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse @@ -49,6 +50,7 @@ class AnthropicTokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ @@ -93,6 +95,12 @@ class AnthropicTokenCounter(BaseTokenCounter): case None: pass + claude_options: Final = claude_count_options( + model_to_use, LlmProviders.ANTHROPIC, request_options, context_management, deployment + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, "anthropic_api") + deployment = deployment or {} litellm_params: Final = deployment.get("litellm_params", {}) api_base: Final = litellm_params.get("api_base") @@ -130,7 +138,8 @@ class AnthropicTokenCounter(BaseTokenCounter): api_base=api_base, tools=tools, system=system, - optional_params=count_options, + optional_params={**count_options, **claude_options.options}, + betas=claude_options.betas, ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..338950c38ea 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -29,12 +29,12 @@ from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject) -COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config") +COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config", "context_management") _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"}) @@ -113,17 +113,24 @@ class ChatCountTranslationError: @with_config(ConfigDict(extra="ignore")) -class _AnthropicChatRequest(TypedDict): - messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]] - system: NotRequired[ReadOnly[JsonValue]] - tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]] +class _CountedOptions(TypedDict): thinking: NotRequired[ReadOnly[JsonValue]] tool_choice: NotRequired[ReadOnly[JsonValue]] output_config: NotRequired[ReadOnly[JsonValue]] output_format: NotRequired[ReadOnly[JsonValue]] + context_management: NotRequired[ReadOnly[JsonValue]] + + +@with_config(ConfigDict(extra="ignore")) +class _AnthropicChatRequest(_CountedOptions): + messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]] + system: NotRequired[ReadOnly[JsonValue]] + tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]] _ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest) +_COUNTED_OPTIONS: Final = TypeAdapter(_CountedOptions) +_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object]) _ChatParams: TypeAlias = dict[str, object] _ChatMessages: TypeAlias = list[AllMessageValues] @@ -240,7 +247,7 @@ def _system_messages(system: object) -> tuple[Mapping[str, object], ...]: return (message,) if isinstance(system, (str, list)) and system else () -def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]: +def _count_options(request: _CountedOptions) -> Mapping[str, JsonValue]: """Structured output reaches chat inference as the beta ``output_format``, which count_tokens refuses; its GA ``output_config.format`` form counts the same schema the same way.""" output_config: Final = request.get("output_config") @@ -256,6 +263,7 @@ def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]: ("thinking", request.get("thinking")), ("tool_choice", request.get("tool_choice")), ("output_config", counted_config or None), + ("context_management", request.get("context_management")), ) if value is not None } @@ -325,6 +333,77 @@ def anthropic_fields_for_chat_tool_use( ) +@dataclass(frozen=True, slots=True) +class ClaudeCountOptions: + options: Mapping[str, JsonValue] + betas: tuple[str, ...] + + +NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=()) +_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1 + + +def claude_count_options( + model: str, + provider: LlmProviders, + request_options: Mapping[str, JsonValue], + context_management: CountContextManagement | None, + deployment: Mapping[str, object] | None, +) -> ClaudeCountOptions | ChatCountTranslationError: + """The thinking, tool_choice, output_config and context_management a Claude count sends, with the + anthropic-beta values they need, after the deployment's own /v1/messages transform rewrites them. + + The transform runs without messages and with a max_tokens above any thinking budget, so it never + caps a budget the caller's real max_tokens would allow. An error it raises is the error inference + answers for the same body. + """ + sent: Final = { + **{name: value for name, value in request_options.items() if value is not None}, + **({} if context_management is None else {"context_management": context_management.spec}), + } + if not sent: + return NO_CLAUDE_COUNT_OPTIONS + from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers + from litellm.exceptions import BadRequestError + from litellm.llms.anthropic.common_utils import AnthropicError, context_management_betas + from litellm.types.router import GenericLiteLLMParams + from litellm.utils import ProviderConfigManager + + messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider) + if messages_config is None: + return NO_CLAUDE_COUNT_OPTIONS + resolved_deployment: Final = deployment or {} + deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + try: + request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts + model=model, + messages=[], + anthropic_messages_optional_request_params={ + **copy.deepcopy(sent), + "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET, + }, + litellm_params=GenericLiteLLMParams.model_validate( + { + "drop_params": deployment_params.get("drop_params"), + "model_info": resolved_deployment.get("model_info"), + } + ), + headers={}, + ) + except (AnthropicError, BadRequestError) as e: + return ChatCountTranslationError(message=str(e.message)) + options: Final = _count_options(_COUNTED_OPTIONS.validate_python(request)) + betas: Final = sorted(context_management_betas(options.get("context_management"))) + return ClaudeCountOptions( + options=options, + betas=tuple( + filter_and_transform_beta_headers(betas, provider.value) + if messages_config.should_filter_anthropic_beta_headers() + else betas + ), + ) + + async def anthropic_count_fields( model: str, messages: Sequence[Mapping[str, object]], @@ -402,7 +481,7 @@ class AnthropicCountTokensConfig: ) ) - def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]: + def get_count_tokens_headers(self, auth_header: Mapping[str, str], betas: Sequence[str] = ()) -> dict[str, str]: """The count-tokens headers around a resolved Anthropic auth header (``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta @@ -412,7 +491,7 @@ class AnthropicCountTokensConfig: "anthropic-version": "2023-06-01", **auth_header, "anthropic-beta": merge_anthropic_beta_headers( - auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION + auth_header.get("anthropic-beta"), (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *betas) ), } diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..54edc273a9e 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -31,6 +31,7 @@ from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( AnthropicError, AnthropicModelInfo, + context_management_betas, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, @@ -792,28 +793,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", ()) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict # Check for structured outputs. Anthropic's newer request shape nests # the schema under output_config.format; the older top-level diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 0f8afaae681..dd0de8559ba 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -4,7 +4,7 @@ Azure AI Anthropic CountTokens API handler. Uses httpx for HTTP requests with Azure authentication. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final import httpx @@ -13,7 +13,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.anthropic.common_utils import AnthropicError, merge_anthropic_beta_headers from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) @@ -38,6 +38,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: object = None, optional_params: Mapping[str, JsonValue] | None = None, + betas: Sequence[str] = (), ) -> dict[str, object]: """ Handle a CountTokens request using httpx with Azure authentication. @@ -101,7 +102,10 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): response: Final = await async_client.post( endpoint_url, - headers=headers, + headers={ + **headers, + "anthropic-beta": merge_anthropic_beta_headers(headers.get("anthropic-beta"), betas), + }, json=request_body, timeout=request_timeout, ) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index af7cf9d81ce..4311b56f9b2 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -15,6 +15,7 @@ from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, ChatCountTranslationError, anthropic_count_fields, + claude_count_options, ) from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, @@ -63,6 +64,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ @@ -107,6 +109,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): case None: pass + claude_options: Final = claude_count_options( + model_to_use, LlmProviders.AZURE_AI, request_options, context_management, deployment + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api") + deployment = deployment or {} litellm_params: Final = deployment.get("litellm_params", {}) @@ -134,7 +142,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): litellm_params=litellm_params, tools=tools, system=system, - optional_params=count_options, + optional_params={**count_options, **claude_options.options}, + betas=claude_options.betas, ) if result is not None: diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..fc84a5aab30 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -42,6 +42,7 @@ class BaseTokenCounter(ABC): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: pass diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 26bc5904e06..18b6caf390f 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -13,7 +13,13 @@ from typing_extensions import assert_never import litellm from litellm._logging import verbose_logger from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields +from litellm.llms.anthropic.count_tokens.transformation import ( + NO_CLAUDE_COUNT_OPTIONS, + ChatCountTranslationError, + ClaudeCountOptions, + anthropic_count_fields, + claude_count_options, +) from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler @@ -74,6 +80,13 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: ) +def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]: + return { + **claude_options.options, + **({"anthropic_beta": list(claude_options.betas)} if claude_options.betas else {}), + } + + def _request_data( model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, system: object ) -> Mapping[str, object]: @@ -143,6 +156,7 @@ class BedrockTokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: if not messages: return None @@ -167,11 +181,27 @@ class BedrockTokenCounter(BaseTokenCounter): ) if isinstance(translated, ChatCountTranslationError): return translated.response(request_model, model_to_use, primary_tokenizer) - runtime_request: Final = ( - _request_data(model_to_use, messages, tools, system) - if translated is None - else _request_data(model_to_use, translated.messages, translated.tools, translated.system) + claude_options: Final = ( + claude_count_options( + counted_model, + LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, + request_options, + context_management, + deployment, + ) + if is_mantle_claude_model(resolved_model) + else NO_CLAUDE_COUNT_OPTIONS ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, primary_tokenizer) + runtime_request: Final = { + **( + _request_data(model_to_use, messages, tools, system) + if translated is None + else _request_data(model_to_use, translated.messages, translated.tools, translated.system) + ), + **_invoke_fields(claude_options), + } mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options} primary: Final = ( diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 34e622b5e8d..5af13ecc4fd 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -26,6 +26,7 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_BETAS: Final = TypeAdapter(tuple[str, ...]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -70,10 +71,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): ) verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url) api_key: Final = litellm_params.get("api_key") + betas: Final = _BETAS.validate_python(request_data.get("anthropic_beta", ())) signed_headers, signed_body = await run_aws_signing( self._sign_request, service_name="bedrock", - headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION}, + headers={ + "Content-Type": "application/json", + "anthropic-version": MANTLE_ANTHROPIC_VERSION, + **({"anthropic-beta": ",".join(betas)} if betas else {}), + }, optional_params=litellm_params, request_data=body, api_base=endpoint_url, diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index bf9c119f6f2..00c7f6093ea 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -19,6 +19,16 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model # max_tokens field; CountTokens only counts input, so it has no effect # on any generation. DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024 +_INVOKE_MODEL_ONLY_FIELDS: Final = frozenset( + ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta") +) + + +def _invoke_model_max_tokens(thinking: JsonValue) -> int: + budget: Final = thinking.get("budget_tokens") if isinstance(thinking, dict) else None + if not isinstance(budget, int): + return DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS + return max(DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, budget + 1) def _json_dict(value: JsonValue) -> dict[str, JsonValue]: @@ -87,6 +97,8 @@ class BedrockCountTokensConfig(BaseAWSLLM): 'converse' or 'invokeModel' """ messages: Final = request_data.get("messages") + if not _INVOKE_MODEL_ONLY_FIELDS.isdisjoint(request_data): + return "invokeModel" if isinstance(messages, list): # Anthropic content blocks carry a "type" key ({"type": "text", ...}); # Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse @@ -200,7 +212,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): # Bedrock validates the body against the model's InvokeModel schema; # Anthropic Messages bodies require these fields. body_data.setdefault("anthropic_version", "bedrock-2023-05-31") - body_data.setdefault("max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS) + body_data.setdefault("max_tokens", _invoke_model_max_tokens(body_data.get("thinking"))) # The CountTokens API expects invokeModel.body as a base64-encoded blob encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode() diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 88c6494e3f7..632eaa21292 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -534,6 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: import copy diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..8953bc6b88d 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ Count tokens using OpenAI's Responses API /input_tokens endpoint. diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 0fa80cb2cad..44bb60573c4 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -30,7 +30,7 @@ from litellm.types.llms.vertex_ai import ( PartType, Schema, ) -from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse from litellm.utils import supports_response_schema, supports_system_messages GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType( @@ -1391,8 +1391,6 @@ class VertexAITokenCounter(BaseTokenCounter): self, custom_llm_provider: str | None = None, ) -> bool: - from litellm.types.utils import LlmProviders - return custom_llm_provider == LlmProviders.VERTEX_AI.value async def _count_translated_claude_request( @@ -1454,6 +1452,7 @@ class VertexAITokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: import copy @@ -1508,6 +1507,17 @@ class VertexAITokenCounter(BaseTokenCounter): if translated_count is not None: return translated_count + from litellm.llms.anthropic.count_tokens.transformation import ( + ChatCountTranslationError, + claude_count_options, + ) + + claude_options: Final = claude_count_options( + model_to_use, LlmProviders.VERTEX_AI, request_options, context_management, deployment + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models") + # Use partner models token counter partner_models_handler: Final = VertexAIPartnerModels() @@ -1538,7 +1548,8 @@ class VertexAITokenCounter(BaseTokenCounter): vertex_credentials=vertex_credentials, system=system, tools=tools, - count_options=count_options, + count_options={**count_options, **claude_options.options}, + betas=claude_options.betas, ) except (PartnerVertexAIError, httpx.HTTPStatusError) as e: status_code: Final = e.response.status_code diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..410d532566b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", []) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict # Check for web search tool for tool in tools: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 4be41e26326..9f0fc69e136 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,6 +6,7 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Sequence from typing import Any, Final from litellm.litellm_core_utils.url_utils import join_api_base @@ -90,6 +91,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): model: str, request_data: dict[str, Any], litellm_params: dict[str, Any], + betas: Sequence[str] = (), ) -> dict[str, Any]: """ Handle token counting request for a Vertex AI partner model. @@ -98,6 +100,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): model: The model name request_data: Request payload (Anthropic Messages API format) litellm_params: LiteLLM parameters containing credentials, project, location + betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header Returns: Dict containing token count information @@ -157,7 +160,10 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): ) # Prepare headers - headers: Final = {"Authorization": f"Bearer {access_token}"} + headers: Final = { + "Authorization": f"Bearer {access_token}", + **({"anthropic-beta": ",".join(betas)} if betas else {}), + } # Get async HTTP client from litellm import LlmProviders diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index bb200fd542d..0a5ac92c733 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -1,6 +1,6 @@ # What is this? ## API Handler for calling Vertex AI Partner Models -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import Final @@ -267,6 +267,7 @@ class VertexAIPartnerModels(VertexBase): system: object | None = None, tools: list[dict[str, object]] | None = None, count_options: Mapping[str, object] = MappingProxyType({}), + betas: Sequence[str] = (), ): """ Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.) @@ -321,6 +322,7 @@ class VertexAIPartnerModels(VertexBase): model=model, request_data=request_data, litellm_params=_litellm_params, + betas=betas, ) return result diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..f5c6f6f48dc 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -336,12 +336,14 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue: _COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params") +_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format") class _CountTokensBodyOptions(TypedDict): output_format: ReadOnly[JsonValue] context_management: ReadOnly[JsonValue] request_litellm_params: ReadOnly[Mapping[str, JsonValue]] + request_options: ReadOnly[Mapping[str, JsonValue]] def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions: @@ -352,6 +354,9 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: "request_litellm_params": MappingProxyType( {"drop_params": True, **sent_params} if auto_drop_params else sent_params ), + "request_options": MappingProxyType( + {key: data[key] for key in _COUNTED_REQUEST_OPTIONS if data.get(key) is not None} + ), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bb8fffd85bc..e228266ba4d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -299,7 +299,7 @@ def generate_feedback_box(): import contextlib from collections import defaultdict from contextlib import AsyncExitStack, asynccontextmanager -from functools import lru_cache, partial +from functools import lru_cache, partial, reduce import litellm import litellm._redis @@ -1374,6 +1374,8 @@ _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody) _LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody) _ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig) _LITELLM_PARAMS_COPY: Final = TypeAdapter(dict[str, object]) +_COUNT_REQUEST_OPTIONS: Final = TypeAdapter(Mapping[str, JsonValue]) +_DROPPED_PARAM_PATHS: Final = TypeAdapter(tuple[object, ...]) _WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings) _UNTYPED_VALUE: Final = TypeAdapter(object) @@ -14810,6 +14812,7 @@ async def _try_provider_token_count( output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> Union["TokenCountResponse", ProviderCountFailure, None]: """Attempt provider-specific token counting. Returns None to fall through to local counting.""" if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): @@ -14837,6 +14840,7 @@ async def _try_provider_token_count( output_format=output_format, context_management=context_management, chat_params=chat_params, + request_options=request_options, ) except httpx.HTTPStatusError as e: return ProviderCountFailure( @@ -15239,6 +15243,20 @@ def _count_context_management( ) +def _count_request_options( + request_options: Mapping[str, JsonValue], + deployment: Mapping[str, object] | None, + sent_params: Mapping[str, object], +) -> Mapping[str, JsonValue]: + from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value + + deployment_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} + dropped: Final = {**deployment_params, **sent_params}.get("additional_drop_params") + dropped_values: Final = _DROPPED_PARAM_PATHS.validate_python(dropped) if isinstance(dropped, list) else () + dropped_paths: Final = tuple(path for path in dropped_values if isinstance(path, str)) + return _COUNT_REQUEST_OPTIONS.validate_python(reduce(delete_nested_value, dropped_paths, dict(request_options))) + + def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> Mapping[str, object]: previous_response_id: Final = None if responses_request is None else responses_request.get("previous_response_id") return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {} @@ -15316,6 +15334,7 @@ async def count_request_tokens( responses_request: Mapping[str, object] | None = None, litellm_call_id: str | None = None, client_oauth_token: str | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse: call_id: Final = resolve_litellm_call_id(litellm_call_id) if request.prompt is None and request.messages is None and request.contents is None: @@ -15335,6 +15354,7 @@ async def count_request_tokens( request_litellm_params=request_litellm_params, responses_request=responses_request, client_oauth_token=client_oauth_token, + request_options=request_options, ) return await logged_token_count( _with_anthropic_content_errors(count) if messages_format == "anthropic" else count, @@ -15377,6 +15397,7 @@ async def _count_request_tokens( request_litellm_params: Mapping[str, JsonValue], responses_request: Mapping[str, object] | None, client_oauth_token: str | None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15484,6 +15505,7 @@ async def _count_request_tokens( output_format=output_format, context_management=_count_context_management(context_management, request.model, request_litellm_params), chat_params=MappingProxyType({}) if chat_count is None else chat_count.params, + request_options=_count_request_options(request_options, count_deployment, sent_params), ), model_to_use=model_to_use, request_model=request.model, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 89352ce8702..cdae81ef9cc 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -13,6 +13,7 @@ import respx from pydantic import JsonValue, TypeAdapter import litellm +from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter @@ -26,7 +27,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse @pytest.mark.parametrize( @@ -475,6 +477,43 @@ _COUNTERS: Final = pytest.mark.parametrize( ) +_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue]) + + +async def _counted_requests( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + messages: list[dict[str, JsonValue]], + tools: list[dict[str, JsonValue]] | None, + messages_format: TokenCountMessagesFormat, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + context_management: CountContextManagement | None = None, + model: str = "claude-opus-5-5", +) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def count(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response(200, json={"input_tokens": 31}) + + with respx.mock: + respx.post(count_endpoint).mock(side_effect=count) + result: Final = await counter_type().count_tokens( + model_to_use=model, + messages=messages, + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + request_model=model, + tools=tools, + messages_format=messages_format, + chat_params=chat_params, + request_options=request_options, + context_management=context_management, + ) + return result, tuple(sent.get() for _ in range(sent.qsize())) + + async def _counted_body( counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str, @@ -483,25 +522,10 @@ async def _counted_body( messages_format: TokenCountMessagesFormat, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]: - sent: Final[SimpleQueue[bytes]] = SimpleQueue() - - def count(request: httpx.Request) -> httpx.Response: - sent.put(request.content) - return httpx.Response(200, json={"input_tokens": 31}) - - with respx.mock: - respx.post(count_endpoint).mock(side_effect=count) - result: Final = await counter_type().count_tokens( - model_to_use="claude-opus-5-5", - messages=messages, - contents=None, - deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, - request_model="claude-opus-5-5", - tools=tools, - messages_format=messages_format, - chat_params=chat_params, - ) - return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize())) + result, requests = await _counted_requests( + counter_type, count_endpoint, messages, tools, messages_format, chat_params + ) + return result, tuple(_JSON_BODY.validate_json(request.content) for request in requests) @pytest.mark.asyncio @@ -842,3 +866,134 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat inference["tool_choice"], inference["tools"], ) + + +_MESSAGE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} +_INFERENCE_PROVIDERS: Final = MappingProxyType( + {AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"} +) +_CONTEXT_BETAS: Final = frozenset( + ( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ) +) +_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA} +_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000} + + +async def _messages_inference_request( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + inference_endpoint: str, + request_options: Mapping[str, JsonValue], +) -> httpx.Request: + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def reply(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response(200, json=_MESSAGE_REPLY) + + with respx.mock: + respx.post(inference_endpoint).mock(side_effect=reply) + await litellm.anthropic_messages( + model=f"{_INFERENCE_PROVIDERS[counter_type]}/claude-haiku-5-5", + messages=[{"role": "user", "content": "Weather in SF?"}], + tools=[_WEATHER_TOOL], + max_tokens=4096, + api_key="sk-ant-api03-test-key", + api_base="https://gateway.example", + **request_options, + ) + return sent.get() + + +def _betas(request: httpx.Request) -> frozenset[str]: + return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + ("request_options", "context_management"), + ( + ({"thinking": _ENABLED_THINKING}, None), + ({"tool_choice": {"type": "any"}}, None), + ({"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, None), + ( + {"thinking": _ENABLED_THINKING}, + {"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]}, + ), + ({}, {"edits": [{"type": "compact_20260112"}]}), + ), + ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"), +) +async def test_counter_sends_the_options_and_context_edits_messages_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + request_options: dict[str, JsonValue], + context_management: dict[str, JsonValue] | None, +) -> None: + """A /v1/messages count that drops thinking, tool_choice, output_config or context_management + counts a different prompt than inference bills, so it carries them the way inference sends them.""" + inference: Final = await _messages_inference_request( + counter_type, + count_endpoint.removesuffix("/count_tokens"), + {**request_options, **({} if context_management is None else {"context_management": context_management})}, + ) + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + request_options=request_options, + context_management=( + None if context_management is None else CountContextManagement(spec=context_management, drop_invalid_edits=False) + ), + model="claude-haiku-5-5", + ) + + inference_body: Final = _JSON_BODY.validate_json(inference.content) + count_body: Final = _JSON_BODY.validate_json(count.content) + inference_output_format: Final = inference_body.get("output_format") + inference_output_config: Final = inference_body.get("output_config") + assert "output_format" not in count_body + assert count_body.get("output_config") == ( + inference_output_config + if inference_output_format is None + else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format} + ) + assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { + name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") + } + assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_refuses_options_messages_inference_refuses_without_calling_anthropic( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + result, requests = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + None, + "anthropic", + request_options={"output_config": {"effort": "fastest"}}, + model="claude-haiku-4-5", + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert requests == () diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index cec7950b859..47484942927 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.utils import CountContextManagement RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" @@ -338,6 +339,105 @@ async def test_claude_count_sends_the_tool_search_tool_the_deployments_inference assert counted_tools == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] +_CONTEXT_EDITS: Final[dict[str, JsonValue]] = { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] +} +_THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = { + "thinking": {"type": "enabled", "budget_tokens": 2000}, + "tool_choice": {"type": "auto"}, +} +_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}} + + +async def _messages_inference_and_count(deployment_model: str, model_to_use: str) -> tuple[_Bedrock, _Bedrock]: + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + tools=[_WEATHER_TOOL], + max_tokens=4096, + context_management=_CONTEXT_EDITS, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + **_THINKING_AND_TOOL_CHOICE, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + result: Final = await _counter(bedrock).count_tokens( + model_to_use=model_to_use, + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}}, + request_model="claude", + tools=[_WEATHER_TOOL], + request_options=_THINKING_AND_TOOL_CHOICE, + context_management=CountContextManagement(spec=_CONTEXT_EDITS, drop_invalid_edits=False), + ) + assert result is not None + assert result.error is False + return inference, bedrock + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "model_to_use"), + ( + ("bedrock/invoke/global.anthropic.claude-haiku-5-5", "global.anthropic.claude-haiku-5-5"), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"), + ), +) +async def test_runtime_count_sends_the_options_and_context_edits_invoke_inference_sends( + deployment_model: str, model_to_use: str +) -> None: + """Bedrock invoke inference drops edits InvokeModel rejects and sends its betas in the body, so a count + that skips either counts a prompt inference never sends, or gets the 400 inference avoids.""" + inference, bedrock = await _messages_inference_and_count(deployment_model, model_to_use) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _invoke_body(bedrock.requests[0]) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + fields: Final = ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta") + assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} + thinking: Final = counted["thinking"] + assert isinstance(thinking, dict) + budget_tokens: Final = thinking.get("budget_tokens", 0) + assert isinstance(budget_tokens, int) and isinstance(counted["max_tokens"], int) + assert counted["max_tokens"] > budget_tokens + + +@pytest.mark.asyncio +async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5" + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + fields: Final = ("thinking", "tool_choice", "output_config", "context_management") + assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} + assert "anthropic_beta" not in counted + assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"] + + +@pytest.mark.asyncio +async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=MESSAGES, + contents=None, + deployment=DEPLOYMENT, + request_model="claude-sonnet-4-5", + request_options={"output_config": {"effort": "fastest"}}, + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert bedrock.posted_hosts() == () + + @pytest.mark.asyncio async def test_claude_chat_tool_use_counted_on_mantle_metadata_route_is_translated_too() -> None: bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 152ab2ffc13..4a9eedb7dbb 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1,8 +1,10 @@ import copy import time -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from contextlib import contextmanager from enum import Enum +from queue import SimpleQueue +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, patch @@ -1840,7 +1842,9 @@ _PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = { async def _partner_counted_bodies( - model: str, chat_params: dict[str, JsonValue] | None = None + model: str, + chat_params: dict[str, JsonValue] | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> tuple[dict[str, object], ...]: from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler @@ -1875,6 +1879,7 @@ async def _partner_counted_bodies( deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, tools=[_PARTNER_RESPONSES_TOOL], chat_params=chat_params or {}, + request_options=request_options, ) assert result is not None @@ -1954,6 +1959,101 @@ async def test_vertex_claude_count_sends_the_structured_output_vertex_inference_ ) +_PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} + + +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None: + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + context_edits: Final[dict[str, JsonValue]] = { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] + } + options: Final[dict[str, JsonValue]] = { + "thinking": {"type": "enabled", "budget_tokens": 2000}, + "tool_choice": {"type": "auto"}, + } + messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}] + tools: Final[list[dict[str, JsonValue]]] = [{"name": "get_weather", "input_schema": {"type": "object"}}] + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def vertex(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response( + 200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY + ) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex)) + with ( + patch.object(handler, "get_async_httpx_client", return_value=client), + patch.object( + VertexBase, "_ensure_access_token_async", new=AsyncMock(return_value=("fake-token", "test-project")) + ), + ): + await litellm.anthropic_messages( + model="vertex_ai/claude-haiku-5-5", + messages=messages, + tools=tools, + max_tokens=4096, + context_management=context_edits, + vertex_project="test-project", + vertex_location="us-east5", + client=client, + **options, + ) + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="claude-haiku-5-5", + messages=messages, + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + tools=tools, + request_options=options, + context_management=CountContextManagement(spec=context_edits, drop_invalid_edits=False), + ) + + assert result is not None + assert result.total_tokens == 37 + inference, count = tuple(sent.get() for _ in range(sent.qsize())) + assert count.url.path.endswith("count-tokens:rawPredict") + counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content) + inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content) + fields: Final = ("thinking", "tool_choice", "output_config", "context_management") + assert {name: counted.get(name) for name in fields} == {name: inferred.get(name) for name in fields} + assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"] + + +@pytest.mark.asyncio +async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="claude-haiku-4-5", + messages=[{"role": "user", "content": "Weather in SF?"}], + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + request_options={"output_config": {"effort": "fastest"}}, + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + + +@pytest.mark.asyncio +async def test_vertex_llama_count_leaves_out_the_claude_only_options() -> None: + model: Final = "meta/llama-4-maverick-17b-128e-instruct-maas" + + assert await _partner_counted_bodies( + model, request_options={"thinking": {"type": "enabled", "budget_tokens": 2000}} + ) == await _partner_counted_bodies(model) + + @pytest.mark.asyncio async def test_vertex_llama_count_sends_chat_tool_use_as_sent() -> None: assert await _partner_counted_bodies("meta/llama-4-maverick-17b-128e-instruct-maas") == ( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 0e8fc6823bd..578343165f7 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -367,6 +367,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + request_options=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -449,6 +450,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + request_options=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -1269,6 +1271,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + request_options=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1328,6 +1331,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + request_options=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1387,6 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + request_options=None, ): raise ProxyException( message="Rate limit exceeded", @@ -1607,6 +1612,103 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}] + +_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = { + "thinking": {"type": "adaptive"}, + "tool_choice": {"type": "auto"}, + "output_config": { + "effort": "low", + "format": { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + }, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, +} + + +def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]: + output_config: Final = body.get("output_config") + return frozenset(name for name in _CLAUDE_COUNT_OPTIONS if name in body) | frozenset( + f"output_config.{key}" for key in (output_config if isinstance(output_config, dict) else {}) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("additional_drop_params", "expected_paths"), + ( + ( + None, + { + "thinking", + "tool_choice", + "output_config", + "output_config.effort", + "output_config.format", + "context_management", + }, + ), + ( + ["tool_choice", "output_config.format"], + {"thinking", "output_config", "output_config.effort", "context_management"}, + ), + ( + ["context_management"], + {"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"}, + ), + ), + ids=("nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management"), +) +async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends( + monkeypatch: pytest.MonkeyPatch, additional_drop_params: list[str] | None, expected_paths: set[str] +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "claude-agent", + "litellm_params": { + "model": "anthropic/claude-haiku-5-5", + "api_key": "sk-ant-test", + "additional_drop_params": additional_drop_params, + }, + } + ] + ) + request: Final = _json_request( + { + "model": "claude-agent", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + **_CLAUDE_COUNT_OPTIONS, + }, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 7 + sent_request: Final = route.calls.last.request + assert _sent_option_paths(json.loads(sent_request.content)) == expected_paths + assert ("context-management-2025-06-27" in sent_request.headers["anthropic-beta"]) == ( + "context_management" in expected_paths + ) + + class _ScriptedCounter(BaseTokenCounter): def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: self._replies = replies @@ -1631,6 +1733,7 @@ class _ScriptedCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: object = None, chat_params: object = None, + request_options: object = None, ) -> TokenCountResponse | None: self.calls.append((model_to_use, messages, tools, system)) self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name")) @@ -3470,7 +3573,7 @@ class _PartnerCountRequest(BaseModel): async def _vertex_partner_word_count( - model: str, request_data: JsonValue, litellm_params: JsonValue + model: str, request_data: JsonValue, litellm_params: JsonValue, betas: object = () ) -> dict[str, JsonValue]: body: Final = _PartnerCountRequest.model_validate(request_data) return {"input_tokens": sum(len(message.content.split()) for message in body.messages)}