diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 578329c4c11..af787b40cd5 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -12,7 +12,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar from urllib.parse import quote, urlparse import httpx -from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError +from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -43,6 +43,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -345,6 +346,18 @@ def _beta_header_values(side: str | Sequence[str] | None) -> tuple[str, ...]: return tuple(entry for entry in side if isinstance(entry, str)) +def context_management_betas(context_management: JsonValue) -> frozenset[str]: + edits: Final = context_management.get("edits") if isinstance(context_management, dict) else None + if not isinstance(edits, (list, tuple)): + return frozenset() + return frozenset( + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value + if isinstance(edit, dict) and edit.get("type") == "compact_20260112" + else ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + for edit in edits + ) + + def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta: str | Sequence[str] | None) -> str: """Merge anthropic-beta header values, deduplicated and sorted. @@ -1283,8 +1296,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index a230294ad3f..c40ae4c86b4 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -4,7 +4,7 @@ Anthropic CountTokens API handler. Uses httpx for HTTP requests instead of the Anthropic SDK. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final import httpx @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + betas: Sequence[str] = (), ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -79,7 +80,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers - headers: Final = self.get_count_tokens_headers(auth_header) + headers: Final = self.get_count_tokens_headers(auth_header, betas) # Use LiteLLM's async httpx client async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index ecf050d185e..a5c19728664 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -2,6 +2,8 @@ Anthropic Token Counter implementation using the CountTokens API. """ +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final from pydantic import JsonValue @@ -10,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler +from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, claude_count_options from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse @@ -38,6 +41,8 @@ class AnthropicTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "anthropic", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: """ Count tokens using Anthropic's CountTokens API. @@ -57,6 +62,12 @@ class AnthropicTokenCounter(BaseTokenCounter): if not messages: return None + claude_options: Final = claude_count_options( + model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, anthropic_beta, deployment + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, "anthropic_api") + deployment = deployment or {} litellm_params: Final = deployment.get("litellm_params", {}) api_base: Final = litellm_params.get("api_base") @@ -84,11 +95,17 @@ class AnthropicTokenCounter(BaseTokenCounter): result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, - messages=messages, + messages=messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], auth_header=auth_header, api_base=api_base, - tools=tools, + tools=tools + if claude_options.messages is None + else [dict(tool) for tool in claude_options.tools or ()] or None, system=system, + optional_params=claude_options.options, + betas=claude_options.betas, ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 8e2688d4360..7516a7aa903 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -4,11 +4,14 @@ Anthropic CountTokens API transformation logic. This module handles the transformation of requests to Anthropic's CountTokens API format. """ +import copy from collections.abc import Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType -from typing import Final +from typing import TYPE_CHECKING, Final -from pydantic import JsonValue, TypeAdapter +from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( @@ -17,12 +20,17 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.types.llms.openai import ChatCompletionImageObject +from litellm.types.utils import LlmProviders, TokenCountResponse + +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject) -COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config") +COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config", "context_management") def _count_image(block: JsonValue) -> JsonValue: @@ -74,6 +82,202 @@ def _lift_leading_system( return conversation, lifted or system +@dataclass(frozen=True, slots=True) +class ChatCountTranslationError: + message: str + + def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse: + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_used, + tokenizer_type=tokenizer_type, + error=True, + error_message=self.message, + status_code=400, + rejected_before_provider_call=True, + ) + + +@with_config(ConfigDict(extra="ignore")) +class _CountedOptions(TypedDict): + thinking: NotRequired[ReadOnly[JsonValue]] + tool_choice: NotRequired[ReadOnly[JsonValue]] + output_config: NotRequired[ReadOnly[JsonValue]] + output_format: NotRequired[ReadOnly[JsonValue]] + context_management: NotRequired[ReadOnly[JsonValue]] + + +@with_config(ConfigDict(extra="ignore")) +class _AnthropicChatRequest(_CountedOptions): + messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]] + tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]] + + +_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest) +_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object]) +_HEADERS: Final = TypeAdapter(dict[str, str]) + + +def _count_options(request: _CountedOptions) -> Mapping[str, JsonValue]: + output_config: Final = request.get("output_config") + output_format: Final = request.get("output_format") + counted_config: Final = { + **(output_config if isinstance(output_config, dict) else {}), + **({} if output_format is None else {"format": output_format}), + } + return MappingProxyType( + { + key: value + for key, value in ( + ("thinking", request.get("thinking")), + ("tool_choice", request.get("tool_choice")), + ("output_config", counted_config or None), + ("context_management", request.get("context_management")), + ) + if value is not None + } + ) + + +@dataclass(frozen=True, slots=True) +class ClaudeCountOptions: + options: Mapping[str, JsonValue] + betas: tuple[str, ...] + messages: tuple[Mapping[str, JsonValue], ...] | None + tools: tuple[Mapping[str, JsonValue], ...] | None + + +NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None) +_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) +_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1 + + +_STATUSES_NOT_CAUSED_BY_THE_REQUEST: Final = frozenset({401, 403, 429}) + + +def is_request_rejection(status_code: int) -> bool: + return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST + + +def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None: + deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}) + deployment_model: Final = deployment_params.get("model") + if not isinstance(deployment_model, str): + return None + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + provider_hint: Final = deployment_params.get("custom_llm_provider") + api_base: Final = deployment_params.get("api_base") + inference_model, _, _, _ = get_llm_provider( + model=deployment_model, + custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + ) + return inference_model + + +def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None: + from litellm.utils import ProviderConfigManager + + messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider) + return None if messages_config is None else (model, messages_config) + + +def _transform_litellm_params(deployment: Mapping[str, object] | None) -> "GenericLiteLLMParams": + from litellm.types.router import GenericLiteLLMParams + + resolved_deployment: Final = deployment or {} + deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + return GenericLiteLLMParams.model_validate( + { + "drop_params": deployment_params.get("drop_params"), + "model_info": resolved_deployment.get("model_info"), + } + ) + + +def _claude_count_options_for( + messages_config: BaseAnthropicMessagesConfig, + model: str, + litellm_params: "GenericLiteLLMParams", + provider: LlmProviders, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + sent: Mapping[str, JsonValue], + anthropic_beta: str | None, +) -> ClaudeCountOptions: + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + + leading_system, conversation = split_leading_system_run(messages) + sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) + sent_params: Final = { + **copy.deepcopy(sent), + **({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}), + } + beta_headers: Final = messages_config.headers_with_anthropic_betas( + {"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation + ) + sent_headers: Final = ( + _HEADERS.validate_python( + update_headers_with_filtered_beta( + headers=dict(beta_headers), + provider=provider.value, + keep=messages_config.model_specific_anthropic_betas(model, litellm_params), + ) + ) + if messages_config.should_filter_anthropic_beta_headers() + else dict(beta_headers) + ) + request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts + model=model, + messages=sent_conversation, + anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET}, + litellm_params=litellm_params, + headers=sent_headers, + ) + transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) + return ClaudeCountOptions( + options=_count_options(transformed), + betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)), + messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]), + tools=transformed.get("tools"), + ) + + +def claude_count_options( + model: str, + provider: LlmProviders, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + request_options: Mapping[str, JsonValue], + anthropic_beta: str | None, + deployment: Mapping[str, object] | None, +) -> ClaudeCountOptions | ChatCountTranslationError: + sent: Final = {name: value for name, value in request_options.items() if value is not None} + if not sent: + return NO_CLAUDE_COUNT_OPTIONS + inference_model: Final = deployment_inference_model(deployment) + transform: Final = ( + None if inference_model is None else _messages_transform(inference_model, provider) + ) or _messages_transform(model, provider) + if transform is None: + return NO_CLAUDE_COUNT_OPTIONS + transform_model, messages_config = transform + litellm_params: Final = _transform_litellm_params(deployment) + from litellm.exceptions import BadRequestError + from litellm.llms.anthropic.common_utils import AnthropicError + + try: + return _claude_count_options_for( + messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta + ) + except (AnthropicError, BadRequestError) as e: + return ChatCountTranslationError(message=str(e.message)) + except (AttributeError, KeyError, TypeError, ValueError) as e: + return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") + + class AnthropicCountTokensConfig: """ Configuration and transformation logic for Anthropic CountTokens API. @@ -137,7 +341,7 @@ class AnthropicCountTokensConfig: ) ) - def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]: + def get_count_tokens_headers(self, auth_header: Mapping[str, str], betas: Sequence[str] = ()) -> dict[str, str]: """The count-tokens headers around a resolved Anthropic auth header (``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta @@ -147,7 +351,7 @@ class AnthropicCountTokensConfig: "anthropic-version": "2023-06-01", **auth_header, "anthropic-beta": merge_anthropic_beta_headers( - auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION + auth_header.get("anthropic-beta"), (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *betas) ), } diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index ad45485fa56..79a02e60fc8 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -26,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( AnthropicError, AnthropicModelInfo, + context_management_betas, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, @@ -378,6 +379,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages=messages, ) + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + return self._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared beta step takes and returns bare dicts + headers=dict(headers), + optional_params=dict(optional_params), + messages=messages, + ) + @staticmethod def _translate_reasoning_effort_to_anthropic( model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str @@ -742,28 +755,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", ()) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict # Check for structured outputs. Anthropic's newer request shape nests # the schema under output_config.format; the older top-level diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 53734a77f3c..dd0de8559ba 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -4,6 +4,7 @@ Azure AI Anthropic CountTokens API handler. Uses httpx for HTTP requests with Azure authentication. """ +from collections.abc import Mapping, Sequence from typing import Final import httpx @@ -12,7 +13,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.anthropic.common_utils import AnthropicError, merge_anthropic_beta_headers from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) @@ -36,6 +37,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): timeout: float | httpx.Timeout | None = None, tools: list[dict[str, JsonValue]] | None = None, system: object = None, + optional_params: Mapping[str, JsonValue] | None = None, + betas: Sequence[str] = (), ) -> dict[str, object]: """ Handle a CountTokens request using httpx with Azure authentication. @@ -66,6 +69,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): messages=messages, tools=tools, system=system, + optional_params=optional_params, ) verbose_logger.debug("Transformed request: %s", request_body) @@ -98,7 +102,10 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): response: Final = await async_client.post( endpoint_url, - headers=headers, + headers={ + **headers, + "anthropic-beta": merge_anthropic_beta_headers(headers.get("anthropic-beta"), betas), + }, json=request_body, timeout=request_timeout, ) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index d1f45751d3e..0f10030a76e 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -3,12 +3,15 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API. """ import os +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final from pydantic import JsonValue import litellm from litellm._logging import verbose_logger +from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, claude_count_options from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, ) @@ -55,6 +58,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "anthropic", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: """ Count tokens using Azure AI Anthropic's CountTokens API. @@ -74,6 +79,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): if not messages: return None + claude_options: Final = claude_count_options( + model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, anthropic_beta, deployment + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api") + deployment = deployment or {} litellm_params: Final = deployment.get("litellm_params", {}) @@ -95,12 +106,18 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): try: result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, - messages=messages, + messages=messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], api_key=api_key, api_base=api_base, litellm_params=litellm_params, - tools=tools, + tools=tools + if claude_options.messages is None + else [dict(tool) for tool in claude_options.tools or ()] or None, system=system, + optional_params=claude_options.options, + betas=claude_options.betas, ) if result is not None: diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index db39e135aca..2b7c900c87a 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx @@ -127,6 +127,18 @@ class BaseAnthropicMessagesConfig(ABC): """ return headers, None + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + return headers + + def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: + beta_header: Final = next((value for name, value in headers.items() if name.lower() == "anthropic-beta"), "") + return tuple(sorted(frozenset(beta.strip() for beta in beta_header.split(",")) - {""})) + def should_filter_anthropic_beta_headers(self) -> bool: """ Whether ``anthropic-beta`` header values should be filtered down to the diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 5ed73e13188..35e898064d5 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -8,6 +8,7 @@ import json from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Final from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals @@ -39,6 +40,8 @@ class BaseTokenCounter(ABC): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: pass diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 068cf4f2e67..624961eb628 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -2,20 +2,30 @@ Bedrock Token Counter implementation using the CountTokens API. """ -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Final from pydantic import JsonValue from typing_extensions import assert_never from litellm._logging import verbose_logger +from litellm.llms.anthropic.count_tokens.transformation import ( + NO_CLAUDE_COUNT_OPTIONS, + ChatCountTranslationError, + ClaudeCountOptions, + claude_count_options, + deployment_inference_model, + is_request_rejection, +) from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER from litellm.types.utils import ( CountContextManagement, LiteLLMPydanticObjectBase, @@ -58,6 +68,48 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: ) +def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options: bool) -> bool: + return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code) + + +def _invoke_fields(options: Mapping[str, JsonValue], betas: Sequence[str]) -> Mapping[str, object]: + return {**options, **({"anthropic_beta": list(betas)} if betas else {})} + + +def _sets_effort(options: Mapping[str, JsonValue]) -> bool: + output_config: Final = options.get("output_config") + return isinstance(output_config, dict) and "effort" in output_config + + +def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]: + if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas: + return claude_options.betas + return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER) + + +def _runs_on_converse(inference_model: str) -> bool: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(inference_model) is None + + +def _options_converse_sends( + request_options: Mapping[str, JsonValue], inference_model: str | None +) -> Mapping[str, JsonValue]: + tool_choice: Final = request_options.get("tool_choice") + if ( + not isinstance(tool_choice, dict) + or "disable_parallel_tool_use" not in tool_choice + or inference_model is None + or not _runs_on_converse(inference_model) + ): + return request_options + return { + **request_options, + "tool_choice": {key: value for key, value in tool_choice.items() if key != "disable_parallel_tool_use"}, + } + + class BedrockTokenCounter(BaseTokenCounter): """Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model""" @@ -77,21 +129,19 @@ class BedrockTokenCounter(BaseTokenCounter): ) -> bool: return custom_llm_provider == LlmProviders.BEDROCK.value - async def _count_with( - self, - handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler, - tokenizer_type: str, - request_data: dict[str, object], - litellm_params: dict[str, object], - resolved_model: str, - ) -> CountTokensOutcome: + def _count_on_mantle( + self, request_data: Mapping[str, object], litellm_params: Mapping[str, object], resolved_model: str + ) -> Awaitable[Mapping[str, JsonValue]]: + return self._mantle_handler.handle_count_tokens_request( + request_data=dict(request_data), + litellm_params=dict(litellm_params), + resolved_model=resolved_model, + client=self._client, + ) + + async def _count_with(self, count: Awaitable[Mapping[str, JsonValue]], tokenizer_type: str) -> CountTokensOutcome: try: - result: Final = await handler.handle_count_tokens_request( - request_data=request_data, - litellm_params=litellm_params, - resolved_model=resolved_model, - client=self._client, - ) + result: Final = await count reply: Final = _CountTokensReply.model_validate(result) except BedrockError as e: verbose_logger.debug( @@ -115,32 +165,74 @@ class BedrockTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "anthropic", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: if not messages: return None + use_mantle: Final = model_to_use.startswith("bedrock_mantle/") + counted_model: Final = model_to_use.removeprefix("bedrock_mantle/") + resolved_model: Final = get_bedrock_base_model(counted_model) + primary_tokenizer: Final = MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE litellm_params: Final = (deployment or {}).get("litellm_params", {}) - request_data: Final[dict[str, object]] = { + claude_options: Final = ( + claude_count_options( + counted_model, + LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, + messages, + tools, + _options_converse_sends(request_options, deployment_inference_model(deployment)), + anthropic_beta, + deployment, + ) + if is_mantle_claude_model(resolved_model) + else NO_CLAUDE_COUNT_OPTIONS + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, primary_tokenizer) + invoke_fields: Final = _invoke_fields(claude_options.options, claude_options.betas) + sent_tools: Final = tools if claude_options.messages is None else claude_options.tools + counted_request: Final[Mapping[str, object]] = { "model": model_to_use, - "messages": messages, - **({"tools": tools} if tools else {}), + "messages": messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], + **({"tools": [dict(tool) for tool in sent_tools]} if sent_tools else {}), **({"system": system} if system else {}), } - use_mantle: Final = model_to_use.startswith("bedrock_mantle/") - resolved_model: Final = get_bedrock_base_model(model_to_use.removeprefix("bedrock_mantle/")) + mantle_request: Final = {**counted_request, **invoke_fields} + runtime_request: Final = { + **counted_request, + **_invoke_fields(claude_options.options, _runtime_count_betas(claude_options)), + } - primary: Final = await self._count_with( - self._mantle_handler if use_mantle else self._runtime_handler, - MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE, - request_data, - litellm_params, - resolved_model, - ) - outcome: Final = ( + primary: Final = ( await self._count_with( - self._mantle_handler, MANTLE_TOKENIZER_TYPE, request_data, litellm_params, resolved_model + self._count_on_mantle(mantle_request, litellm_params, resolved_model), primary_tokenizer + ) + if use_mantle + else await self._count_with( + self._runtime_handler.handle_count_tokens_request( + request_data=runtime_request, + litellm_params=litellm_params, + resolved_model=resolved_model, + client=self._client, + input_type="invokeModel" if invoke_fields else None, + ), + primary_tokenizer, + ) + ) + fallback: Final = ( + await self._count_with( + self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE ) if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) + else None + ) + outcome: Final = ( + fallback + if fallback is not None and _mantle_answered_for_runtime(fallback, forwarded_options=bool(invoke_fields)) else primary ) match outcome: diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 3bae1f22490..6f684121d6e 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -12,7 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.llms.bedrock.base_aws_llm import run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig +from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig, BedrockCountTokensInputType from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client @@ -29,6 +29,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): litellm_params: dict[str, Any], resolved_model: str, client: AsyncHTTPHandler | None = None, + input_type: BedrockCountTokensInputType | None = None, ) -> dict[str, Any]: """ Handle a CountTokens request using existing LiteLLM patterns. @@ -43,7 +44,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): """ try: # Validate the request - self.validate_count_tokens_request(request_data) + self.validate_count_tokens_request(request_data, input_type) verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model) @@ -57,7 +58,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): verbose_logger.debug("Retrieved AWS region: %s", aws_region_name) # Transform request to Bedrock format (supports both Converse and InvokeModel) - bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data) + bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens( + request_data=request_data, input_type=input_type + ) verbose_logger.debug("Transformed request: %s", bedrock_request) diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 4ea4440a3c4..c6bec801736 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import httpx @@ -6,7 +7,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountTokensConfig +from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client @@ -23,6 +24,8 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) +_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_BETAS: Final = TypeAdapter(tuple[str, ...]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -61,13 +64,21 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): messages=request["messages"], tools=request.get("tools"), system=request.get("system"), + optional_params=_COUNT_OPTIONS.validate_python( + {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES} + ), ) verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url) api_key: Final = litellm_params.get("api_key") + betas: Final = _BETAS.validate_python(request_data.get("anthropic_beta", ())) signed_headers, signed_body = await run_aws_signing( self._sign_request, service_name="bedrock", - headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION}, + headers={ + "Content-Type": "application/json", + "anthropic-version": MANTLE_ANTHROPIC_VERSION, + **({"anthropic-beta": ",".join(betas)} if betas else {}), + }, optional_params=litellm_params, request_data=body, api_base=endpoint_url, diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index 48fc41ed12b..a88ec69e6a3 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -7,7 +7,7 @@ to AWS Bedrock's CountTokens API format and vice versa. import re from collections.abc import Mapping -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from pydantic import JsonValue @@ -18,6 +18,14 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model # max_tokens field; CountTokens only counts input, so it has no effect # on any generation. DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024 +BedrockCountTokensInputType: TypeAlias = Literal["converse", "invokeModel"] + + +def _invoke_model_max_tokens(thinking: JsonValue) -> int: + budget: Final = thinking.get("budget_tokens") if isinstance(thinking, dict) else None + if not isinstance(budget, int): + return DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS + return max(DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, budget + 1) def _json_dict(value: JsonValue) -> dict[str, JsonValue]: @@ -72,7 +80,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): - Response: {"inputTokens": } """ - def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> Literal["converse", "invokeModel"]: + def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> BedrockCountTokensInputType: """ Detect whether to use 'converse' or 'invokeModel' input format. @@ -103,6 +111,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): def transform_anthropic_to_bedrock_count_tokens( self, request_data: Mapping[str, JsonValue], + input_type: BedrockCountTokensInputType | None = None, ) -> dict[str, JsonValue]: """ Transform request to Bedrock CountTokens format. @@ -133,9 +142,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): } } """ - input_type: Final = self._detect_input_type(request_data) - - if input_type == "converse": + if (input_type or self._detect_input_type(request_data)) == "converse": return self._transform_to_converse_format(request_data) else: return self._transform_to_invoke_model_format(request_data) @@ -196,7 +203,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): # Bedrock validates the body against the model's InvokeModel schema; # Anthropic Messages bodies require these fields. body_data.setdefault("anthropic_version", "bedrock-2023-05-31") - body_data.setdefault("max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS) + body_data.setdefault("max_tokens", _invoke_model_max_tokens(body_data.get("thinking"))) # The CountTokens API expects invokeModel.body as a base64-encoded blob encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode() @@ -257,23 +264,20 @@ class BedrockCountTokensConfig(BaseAWSLLM): return {"input_tokens": input_tokens} - def validate_count_tokens_request(self, request_data: Mapping[str, JsonValue]) -> None: + def validate_count_tokens_request( + self, request_data: Mapping[str, JsonValue], input_type: BedrockCountTokensInputType | None = None + ) -> None: """ Validate the incoming count tokens request. Supports both Converse and InvokeModel input formats. - Args: - request_data: The request payload - Raises: ValueError: If the request is invalid """ if not request_data.get("model"): raise ValueError("model parameter is required") - input_type: Final = self._detect_input_type(request_data) - - if input_type == "converse": + if (input_type or self._detect_input_type(request_data)) == "converse": # Validate Converse format (messages-based) messages: Final = request_data.get("messages", []) if not messages: diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index d428b83451c..bcd79fcde60 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast @@ -77,6 +77,7 @@ else: _MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) class AmazonAnthropicClaudeMessagesConfig( @@ -100,6 +101,17 @@ class AmazonAnthropicClaudeMessagesConfig( def beta_headers_provider(self) -> str: return self.custom_llm_provider or "bedrock" + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + return headers + + def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: + return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ()) + def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]: if bedrock_supports_per_turn_control( model, _MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)) diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index ca93cc3e617..04ea0c295c7 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -18,6 +18,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, @@ -67,6 +68,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): def should_filter_anthropic_beta_headers(self) -> bool: return False + def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: + return BaseAnthropicMessagesConfig.sent_anthropic_betas(self, headers, request) + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 4006c89954d..25fb503158d 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -3,6 +3,7 @@ import datetime import json import math from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import Any, Final import httpx @@ -498,6 +499,8 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: import copy diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 7ff73109aad..3140518bd18 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -3,6 +3,7 @@ OpenAI Token Counter implementation using the Responses API /input_tokens endpoi """ from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import Any, Final import openai @@ -96,6 +97,8 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: """ Count tokens using OpenAI's Responses API /input_tokens endpoint. diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3c9a8429882..5441fd4deb7 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -23,7 +23,7 @@ from litellm.types.llms.vertex_ai import ( PartType, Schema, ) -from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse from litellm.utils import supports_response_schema, supports_system_messages GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType( @@ -1259,8 +1259,6 @@ class VertexAITokenCounter(BaseTokenCounter): self, custom_llm_provider: str | None = None, ) -> bool: - from litellm.types.utils import LlmProviders - return custom_llm_provider == LlmProviders.VERTEX_AI.value async def count_tokens( @@ -1275,6 +1273,8 @@ class VertexAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> TokenCountResponse | None: import copy @@ -1313,6 +1313,24 @@ class VertexAITokenCounter(BaseTokenCounter): # Check if this is a partner model (Claude, Mistral, etc.) if VertexAIPartnerModels.is_vertex_partner_model(model_to_use): + from litellm.llms.anthropic.count_tokens.transformation import ( + ChatCountTranslationError, + claude_count_options, + ) + + sent_messages: Final = messages or [] + claude_options: Final = claude_count_options( + model_to_use, + LlmProviders.VERTEX_AI, + sent_messages, + tools, + request_options, + anthropic_beta, + deployment, + ) + if isinstance(claude_options, ChatCountTranslationError): + return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models") + # Use partner models token counter partner_models_handler: Final = VertexAIPartnerModels() @@ -1336,13 +1354,19 @@ class VertexAITokenCounter(BaseTokenCounter): try: result = await partner_models_handler.count_tokens( model=model_to_use, - messages=messages or [], + messages=sent_messages + if claude_options.messages is None + else [dict(message) for message in claude_options.messages], litellm_params=partner_litellm_params, vertex_project=vertex_project, vertex_location=vertex_location, vertex_credentials=vertex_credentials, system=system, - tools=tools, + tools=tools + if claude_options.messages is None + else [dict(tool) for tool in claude_options.tools or ()] or None, + count_options=claude_options.options, + betas=claude_options.betas, ) except (PartnerVertexAIError, httpx.HTTPStatusError) as e: status_code: Final = e.response.status_code diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 5d748ad6e74..87010710da1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -1,6 +1,8 @@ +from collections.abc import Mapping, Sequence from typing import Any, Final -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -17,6 +19,11 @@ from ....vertex_llm_base import VertexBase from ..output_params_utils import sanitize_vertex_anthropic_output_params +def _is_web_search(tool: Mapping[str, object]) -> bool: + tool_type: Final = tool.get("type") + return isinstance(tool_type, str) and tool_type.startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value) + + class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase): @property def custom_llm_provider(self) -> str | None: @@ -66,59 +73,47 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers["content-type"] = "application/json" - # Add beta headers for Vertex AI - tools: Final = optional_params.get("tools", []) - beta_values: Final[set[str]] = set() + return dict(self.headers_with_anthropic_betas(headers, optional_params, messages)), api_base - # Get existing beta headers if any - existing_beta: Final = headers.get("anthropic-beta") - if existing_beta: - beta_values.update(b.strip() for b in existing_beta.split(",")) - - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", []) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) - - # Check for web search tool - for tool in tools: - if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value): - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value) - break - - # Check for tool search tools - Vertex AI uses different beta header - anthropic_model_info: Final = AnthropicModelInfo() - if anthropic_model_info.is_tool_search_used(tools): - beta_values.add(get_tool_search_beta_header("vertex_ai")) - - if optional_params.get("safeguards") is not None: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value) - - if messages_carry_output_config(messages): - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) - - if beta_values: - headers["anthropic-beta"] = ",".join(beta_values) - - return headers, api_base + def headers_with_anthropic_betas( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + messages: Sequence[object], + ) -> Mapping[str, str]: + tools: Final = tuple( + tool for tool in map(as_str_mapping, iter_items(optional_params.get("tools") or ())) if tool is not None + ) + existing_betas: Final = (headers.get("anthropic-beta") or "").split(",") + beta_values: Final = frozenset( + ( + *(beta.strip() for beta in existing_betas if beta.strip()), + *context_management_betas(optional_params.get("context_management")), # pyright: ignore[reportArgumentType] # optional_params is the config contract's untyped request body + *( + (ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value,) + if any(map(_is_web_search, tools)) + else () + ), + *( + (get_tool_search_beta_header("vertex_ai"),) + if AnthropicModelInfo().is_tool_search_used(list(tools)) + else () + ), + *( + (ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,) + if optional_params.get("safeguards") is not None + else () + ), + *( + (ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value,) + if messages_carry_output_config(messages) + else () + ), + ) + ) + if not beta_values: + return headers + return {**headers, "anthropic-beta": ",".join(sorted(beta_values))} def get_complete_url( self, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 7100c9efde3..020d578eafd 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,6 +6,7 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Sequence from typing import Any, Final from litellm.llms.custom_httpx.http_handler import get_async_httpx_client @@ -91,6 +92,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): model: str, request_data: dict[str, Any], litellm_params: dict[str, Any], + betas: Sequence[str] = (), ) -> dict[str, Any]: """ Handle token counting request for a Vertex AI partner model. @@ -158,7 +160,10 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): ) # Prepare headers - headers: Final = {"Authorization": f"Bearer {access_token}"} + headers: Final = { + "Authorization": f"Bearer {access_token}", + **({"anthropic-beta": ",".join(betas)} if betas else {}), + } # Get async HTTP client from litellm import LlmProviders diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index c855a073648..0a5ac92c733 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -1,6 +1,6 @@ # What is this? ## API Handler for calling Vertex AI Partner Models -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import Final @@ -266,6 +266,8 @@ class VertexAIPartnerModels(VertexBase): vertex_credentials=None, system: object | None = None, tools: list[dict[str, object]] | None = None, + count_options: Mapping[str, object] = MappingProxyType({}), + betas: Sequence[str] = (), ): """ Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.) @@ -302,6 +304,7 @@ class VertexAIPartnerModels(VertexBase): **MappingProxyType( {key: value for key, value in (("system", system), ("tools", tools)) if value is not None} ), + **count_options, } # Prepare litellm_params with credentials @@ -319,6 +322,7 @@ class VertexAIPartnerModels(VertexBase): model=model, request_data=request_data, litellm_params=_litellm_params, + betas=betas, ) return result diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index a763bb3a341..33391c150b5 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -50,7 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( with_litellm_call_id, ) from litellm.proxy.common_utils.token_count_settings import count_source_fields -from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, should_auto_drop_params_for_agentic_cli from litellm.router import Router router: Final = APIRouter() @@ -333,12 +333,14 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue: _COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params") +_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format", "context_management") class _CountTokensBodyOptions(TypedDict): output_format: ReadOnly[JsonValue] context_management: ReadOnly[JsonValue] request_litellm_params: ReadOnly[Mapping[str, JsonValue]] + request_options: ReadOnly[Mapping[str, JsonValue]] def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions: @@ -349,6 +351,9 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: "request_litellm_params": MappingProxyType( {"drop_params": True, **sent_params} if auto_drop_params else sent_params ), + "request_options": MappingProxyType( + {key: data[key] for key in _COUNTED_REQUEST_OPTIONS if data.get(key) is not None} + ), } @@ -381,7 +386,7 @@ async def count_tokens( Returns: {"input_tokens": } """ - from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config + from litellm.proxy.proxy_server import count_request_tokens, general_settings_view, llm_router, proxy_config litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) try: @@ -412,6 +417,11 @@ async def count_tokens( request=token_request, call_endpoint=True, messages_format="anthropic", + anthropic_beta=request.headers.get("anthropic-beta"), + forward_client_headers=LiteLLMProxyRequestSetup.forwards_client_headers_for_all_models( + general_settings_view() + ) + or LiteLLMProxyRequestSetup.forwards_client_headers_for_model_group(token_request.model, user_api_key_dict), **_count_tokens_body_options(data, auto_drop_params), ) except UnsupportedContentTypeError as e: diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 22b35bd1001..8e556678c98 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.llms.anthropic.count_tokens.transformation import is_request_rejection from litellm.proxy._types import ProxyException from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer from litellm.types.utils import TokenCountResponse @@ -66,6 +67,10 @@ class ProviderCountFailure: def cannot_count_model(self) -> bool: return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES + @property + def rejected_request(self) -> bool: + return is_request_rejection(self.status_code) + def with_hint(self, hint: str) -> "ProviderCountFailure": return replace(self, message=_with_hint(self.message, hint)) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0645cc43ed9..53c513cf1c6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1527,27 +1527,33 @@ class LiteLLMProxyRequestSetup: return returned_headers @staticmethod - def add_headers_to_llm_call_by_model_group(data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth) -> dict: - """ - Add headers to the LLM call by model group - """ + def forwards_client_headers_for_all_models(general_settings: Mapping[str, object] | None) -> bool: + return general_settings is not None and general_settings.get("forward_client_headers_to_llm_api") is True + + @staticmethod + def forwards_client_headers_for_model_group(model: str | None, user_api_key_dict: UserAPIKeyAuth) -> bool: from litellm.proxy.auth.auth_checks import check_model_access_helper from litellm.proxy.proxy_server import llm_router - data_model: Final = data.get("model") - - if ( - data_model is not None + return ( + model is not None and litellm.model_group_settings is not None and litellm.model_group_settings.forward_client_headers_to_llm_api is not None and check_model_access_helper( - model=data_model, + model=model, llm_router=llm_router, models=litellm.model_group_settings.forward_client_headers_to_llm_api, team_model_aliases=user_api_key_dict.team_model_aliases, team_id=user_api_key_dict.team_id, ) # handles aliases, wildcards, etc. - ): + ) + + @staticmethod + def add_headers_to_llm_call_by_model_group(data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth) -> dict: + """ + Add headers to the LLM call by model group + """ + if LiteLLMProxyRequestSetup.forwards_client_headers_for_model_group(data.get("model"), user_api_key_dict): _headers: Final = LiteLLMProxyRequestSetup.add_headers_to_llm_call(headers, user_api_key_dict) if _headers != {}: data["headers"] = _headers @@ -1584,7 +1590,7 @@ class LiteLLMProxyRequestSetup: """ data: Final = LitellmDataForBackendLLMCall() - if general_settings and general_settings.get("forward_client_headers_to_llm_api") is True: + if LiteLLMProxyRequestSetup.forwards_client_headers_for_all_models(general_settings): _headers: Final = LiteLLMProxyRequestSetup.add_headers_to_llm_call(headers, user_api_key_dict) if _headers != {}: data["headers"] = _headers @@ -4038,14 +4044,19 @@ async def add_guardrails_from_policy_engine( ) -_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( - ( - LlmProviders.ANTHROPIC.value, - LlmProviders.BEDROCK.value, - LlmProviders.BEDROCK_MANTLE.value, - LlmProviders.VERTEX_AI.value, - ) +_ANTHROPIC_API_HEADER_PROVIDER_NAMES: Final = ( + LlmProviders.ANTHROPIC.value, + LlmProviders.BEDROCK.value, + LlmProviders.BEDROCK_MANTLE.value, + LlmProviders.VERTEX_AI.value, ) +_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(_ANTHROPIC_API_HEADER_PROVIDER_NAMES) + + +def receives_anthropic_api_headers(custom_llm_provider: str | None) -> bool: + return custom_llm_provider in _ANTHROPIC_API_HEADER_PROVIDER_NAMES + + _ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 80970cba90b..0e64b2c5433 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -293,7 +293,7 @@ def generate_feedback_box(): import contextlib from collections import defaultdict from contextlib import AsyncExitStack, asynccontextmanager -from functools import lru_cache, partial +from functools import lru_cache, partial, reduce import litellm import litellm._redis @@ -640,7 +640,11 @@ from litellm.proxy.list_api.common import ( problem_response, request_validation_problem, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + add_litellm_data_to_request, + receives_anthropic_api_headers, +) from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, ) @@ -1316,6 +1320,8 @@ class _WebsearchInterceptionLitellmSettings(TypedDict, total=False): _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody) _LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody) _ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig) +_COUNT_REQUEST_OPTIONS: Final = TypeAdapter(Mapping[str, JsonValue]) +_DROPPED_PARAM_PATHS: Final = TypeAdapter(tuple[object, ...]) _WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings) _UNTYPED_VALUE: Final = TypeAdapter(object) @@ -14279,6 +14285,8 @@ async def _try_provider_token_count( messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, ) -> Union["TokenCountResponse", ProviderCountFailure, None]: """Attempt provider-specific token counting. Returns None to fall through to local counting.""" if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): @@ -14305,6 +14313,8 @@ async def _try_provider_token_count( messages_format=messages_format, output_format=output_format, context_management=context_management, + request_options=request_options, + anthropic_beta=anthropic_beta, ) except httpx.HTTPStatusError as e: return ProviderCountFailure( @@ -14358,6 +14368,33 @@ async def _provider_token_count( raise failure.to_proxy_exception() +def _retried_without_rejected_options( + count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]", + request_options: Mapping[str, JsonValue], +) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]: + if not request_options: + return count_with + return partial(_count_retrying_without_options, count_with, tuple(request_options)) + + +async def _count_retrying_without_options( + count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]", + forwarded_options: tuple[str, ...], + model: str, + region: str | None, +) -> Union["TokenCountResponse", ProviderCountFailure, None]: + counted: Final = await count_with(model, region) + if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): + return counted + verbose_proxy_logger.warning( + "Token count for %s %s with the request's %s, so it counts again without them and the caller's betas", + model, + "could not be built" if counted.rejected_before_provider_call else f"answered {counted.status_code}", + ", ".join(forwarded_options), + ) + return await count_with(model, region, request_options=MappingProxyType({})) + + @dataclass(frozen=True, slots=True) class _LocalTokenEstimate: total_tokens: int @@ -14682,6 +14719,33 @@ def _count_context_management( ) +_STRING_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +def _deployment_with_request_drop_params( + deployment: Mapping[str, object] | None, sent_params: Mapping[str, object] +) -> Mapping[str, object] | None: + if "drop_params" not in sent_params: + return deployment + resolved_deployment: Final = deployment or {} + litellm_params: Final = _STRING_KEYED_MAPPING.validate_python(resolved_deployment.get("litellm_params") or {}) + return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": sent_params["drop_params"]}} + + +def _count_request_options( + request_options: Mapping[str, JsonValue], + deployment: Mapping[str, object] | None, + sent_params: Mapping[str, object], +) -> Mapping[str, JsonValue]: + from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value + + deployment_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} + dropped: Final = {**deployment_params, **sent_params}.get("additional_drop_params") + dropped_values: Final = _DROPPED_PARAM_PATHS.validate_python(dropped) if isinstance(dropped, list) else () + dropped_paths: Final = tuple(path for path in dropped_values if isinstance(path, str)) + return _COUNT_REQUEST_OPTIONS.validate_python(reduce(delete_nested_value, dropped_paths, dict(request_options))) + + def _system_message(system: object) -> ChatCompletionSystemMessage | None: if not isinstance(system, (str, list)) or not system: return None @@ -14714,6 +14778,9 @@ async def count_request_tokens( output_format: JsonValue = None, context_management: JsonValue = None, request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}), + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, + forward_client_headers: bool = False, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -14781,23 +14848,40 @@ async def count_request_tokens( prompt_only: Final = bool(prompt) and conversation_empty contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents)) prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES + sent_params: Final[Mapping[str, object]] = { + **(_router_default_litellm_params(llm_router) if llm_router is not None else {}), + **request_litellm_params, + } + counted_options: Final = _count_request_options(request_options, deployment_view, sent_params) ladder: Final = ( None if provider_counter is None else await _provider_token_count( - count_with=partial( - _try_provider_token_count, - provider_counter, - custom_llm_provider, - messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages, - contents=None if prompt_as_message else contents, - deployment=deployment, - request_model=request.model, - tools=tools, - system=system, - messages_format=messages_format, - output_format=output_format, - context_management=_count_context_management(context_management, request.model, request_litellm_params), + count_with=_retried_without_rejected_options( + partial( + _try_provider_token_count, + provider_counter, + custom_llm_provider, + messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages, + contents=None if prompt_as_message else contents, + deployment=_deployment_with_request_drop_params( + None if deployment is None else _STRING_KEYED_MAPPING.validate_python(deployment), + sent_params, + ), + request_model=request.model, + tools=tools, + system=system, + messages_format=messages_format, + output_format=output_format, + context_management=_count_context_management( + context_management, request.model, request_litellm_params + ), + request_options=counted_options, + anthropic_beta=anthropic_beta + if forward_client_headers or receives_anthropic_api_headers(custom_llm_provider) + else None, + ), + counted_options, ), model_to_use=model_to_use, settings=settings, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 9af7497bc82..f486e5b58f9 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -1,7 +1,10 @@ import asyncio from base64 import b64encode +from collections.abc import Mapping from copy import deepcopy +from queue import SimpleQueue from threading import get_ident +from types import MappingProxyType from typing import Final import httpx @@ -10,16 +13,25 @@ import respx from pydantic import JsonValue, TypeAdapter import litellm +from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler +from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, ) +from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) +from litellm.types.llms.anthropic import ( + ANTHROPIC_ADVISOR_TOOL_TYPE, + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) +from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @pytest.mark.parametrize( @@ -441,3 +453,337 @@ async def test_handler_sends_the_leading_system_run_as_system_not_as_a_message(h "system": [{"type": "text", "text": "Be terse"}], "messages": [{"role": "user", "content": "Hello, how are you?"}], } + + +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {"type": "object", "properties": {"city": {"type": "string"}}} +_ANSWER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, +} + + +_COUNTERS: Final = pytest.mark.parametrize( + ("counter_type", "count_endpoint"), + ( + (AnthropicTokenCounter, "https://gateway.example/v1/messages/count_tokens"), + (AzureAIAnthropicTokenCounter, "https://gateway.example/anthropic/v1/messages/count_tokens"), + ), +) + + +_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue]) + + +async def _counted_requests( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + messages: list[dict[str, JsonValue]], + tools: list[dict[str, JsonValue]] | None, + messages_format: TokenCountMessagesFormat, + request_options: Mapping[str, JsonValue] = MappingProxyType({}), + anthropic_beta: str | None = None, + model: str = "claude-opus-5-5", +) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def count(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response(200, json={"input_tokens": 31}) + + with respx.mock: + respx.post(count_endpoint).mock(side_effect=count) + result: Final = await counter_type().count_tokens( + model_to_use=model, + messages=messages, + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + request_model=model, + tools=tools, + messages_format=messages_format, + request_options=request_options, + anthropic_beta=anthropic_beta, + ) + return result, tuple(sent.get() for _ in range(sent.qsize())) + + +_MESSAGE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} +_INFERENCE_PROVIDERS: Final = MappingProxyType( + {AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"} +) +_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA} +_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000} + + +async def _messages_inference_request( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + inference_endpoint: str, + request_options: Mapping[str, JsonValue], + model: str = "claude-haiku-5-5", + extra_headers: Mapping[str, str] = MappingProxyType({}), +) -> httpx.Request: + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def reply(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response(200, json=_MESSAGE_REPLY) + + with respx.mock: + respx.post(inference_endpoint).mock(side_effect=reply) + await litellm.anthropic_messages( + model=f"{_INFERENCE_PROVIDERS[counter_type]}/{model}", + messages=[{"role": "user", "content": "Weather in SF?"}], + tools=[_WEATHER_TOOL], + max_tokens=4096, + api_key="sk-ant-api03-test-key", + api_base="https://gateway.example", + extra_headers=dict(extra_headers), + **request_options, + ) + return sent.get() + + +def _on_both_counters( + *betas: str, +) -> Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]]: + return MappingProxyType({AnthropicTokenCounter: frozenset(betas), AzureAIAnthropicTokenCounter: frozenset(betas)}) + + +def _betas(request: httpx.Request) -> frozenset[str]: + return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") +@_COUNTERS +@pytest.mark.parametrize( + ("request_options", "expected_betas"), + ( + ({"thinking": _ENABLED_THINKING}, _on_both_counters()), + ({"tool_choice": {"type": "any"}}, _on_both_counters()), + ( + {"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, + _on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value), + ), + ( + { + "thinking": _ENABLED_THINKING, + "context_management": { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] + }, + }, + _on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value), + ), + ( + {"context_management": {"edits": [{"type": "compact_20260112"}]}}, + MappingProxyType( + { + AnthropicTokenCounter: frozenset((ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,)), + AzureAIAnthropicTokenCounter: frozenset(), + } + ), + ), + ), + ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"), +) +async def test_counter_sends_the_options_and_context_edits_messages_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + request_options: dict[str, JsonValue], + expected_betas: Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]], +) -> None: + inference: Final = await _messages_inference_request( + counter_type, count_endpoint.removesuffix("/count_tokens"), request_options + ) + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + request_options=request_options, + model="claude-haiku-5-5", + ) + + inference_body: Final = _JSON_BODY.validate_json(inference.content) + count_body: Final = _JSON_BODY.validate_json(count.content) + inference_output_format: Final = inference_body.get("output_format") + inference_output_config: Final = inference_body.get("output_config") + assert "output_format" not in count_body + assert count_body.get("output_config") == ( + inference_output_config + if inference_output_format is None + else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format} + ) + assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { + name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") + } + assert expected_betas[counter_type] == _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_sends_the_thinking_display_and_caller_betas_messages_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + caller_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value + display_thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}} + inference: Final = await _messages_inference_request( + counter_type, + count_endpoint.removesuffix("/count_tokens"), + display_thinking, + model="claude-opus-4-6", + extra_headers={"anthropic-beta": caller_beta}, + ) + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + request_options=display_thinking, + anthropic_beta=caller_beta, + model="claude-opus-4-6", + ) + + counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} + assert counted_betas == _betas(inference) + assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_without_options_sends_only_the_token_counting_beta_and_the_bare_count_body( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + anthropic_beta=ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value, + model="claude-haiku-5-5", + ) + + assert _betas(count) == {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} + assert _JSON_BODY.validate_json(count.content) == { + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "tools": [_WEATHER_TOOL], + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_sending_options_still_counts_the_leading_system_run_as_system( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "system", "content": "Be terse."}, {"role": "user", "content": "Weather in SF?"}], + None, + "anthropic", + request_options={"thinking": _ENABLED_THINKING}, + model="claude-haiku-5-5", + ) + + body: Final = _JSON_BODY.validate_json(count.content) + assert "thinking" in body + assert {name: body.get(name) for name in ("system", "messages")} == { + "system": [{"type": "text", "text": "Be terse."}], + "messages": [{"role": "user", "content": "Weather in SF?"}], + } + + +_ADVISOR_TOOL: Final[dict[str, JsonValue]] = { + "type": ANTHROPIC_ADVISOR_TOOL_TYPE, + "name": "advisor", + "model": "claude-opus-5-5", +} +_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}} +_ADVISOR_RESULT: Final[dict[str, JsonValue]] = { + "type": "advisor_tool_result", + "tool_use_id": "srvtoolu_1", + "content": {"type": "advisor_result", "text": "Check the forecast first."}, +} +_ADVISED_ANSWER: Final[dict[str, JsonValue]] = {"type": "text", "text": "It is sunny."} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + ("tools", "counted_reply"), + [ + pytest.param( + [_ADVISOR_TOOL, _WEATHER_TOOL], + [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER], + id="advisor_tool_sent", + ), + pytest.param([_WEATHER_TOOL], [_ADVISED_ANSWER], id="advisor_tool_not_sent"), + ], +) +async def test_counter_sending_options_keeps_advisor_blocks_only_when_the_advisor_tool_is_sent( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + tools: list[dict[str, JsonValue]], + counted_reply: list[JsonValue], +) -> None: + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [ + {"role": "user", "content": "Weather in SF?"}, + {"role": "assistant", "content": [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER]}, + {"role": "user", "content": "And tomorrow?"}, + ], + tools, + "anthropic", + request_options={"thinking": _ENABLED_THINKING}, + model="claude-haiku-5-5", + ) + + body: Final = _JSON_BODY.validate_json(count.content) + assert {name: body.get(name) for name in ("messages", "tools")} == { + "messages": [ + {"role": "user", "content": "Weather in SF?"}, + {"role": "assistant", "content": counted_reply}, + {"role": "user", "content": "And tomorrow?"}, + ], + "tools": tools, + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_refuses_options_messages_inference_refuses_without_calling_anthropic( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + result, requests = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + None, + "anthropic", + request_options={"output_config": {"effort": "fastest"}}, + model="claude-haiku-4-5", + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert requests == () diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py index b357c5ac126..5ed45da3c04 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py @@ -1,5 +1,8 @@ import base64 import json +from typing import Final + +import pytest from litellm.llms.bedrock.count_tokens.transformation import ( DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, @@ -53,6 +56,28 @@ def test_detect_input_type_converse_blocks_route_to_converse(): assert config._detect_input_type(request) == "converse" +@pytest.mark.parametrize( + "option", + ( + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + {"tool_choice": {"type": "auto"}}, + {"output_config": {"effort": "low"}}, + ), +) +def test_pass_through_body_with_string_content_is_counted_as_converse_whatever_options_it_carries( + option: dict[str, object], +) -> None: + request: Final = { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "messages": [{"role": "user", "content": "hi"}], + **option, + } + + result: Final = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request) + + assert result["input"] == {"converse": {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}} + + def test_transform_to_invoke_model_format_base64_encodes_body(): """The CountTokens API expects invokeModel.body as a base64-encoded blob; Anthropic Messages bodies additionally need anthropic_version/max_tokens diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 8749b01ed5f..b729aa2a4e4 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -1,12 +1,20 @@ -from collections.abc import Mapping +import base64 +import json +from collections.abc import Callable, Mapping from typing import Final import httpx import pytest from pydantic import JsonValue, TypeAdapter +import litellm from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_EFFORT_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" @@ -251,3 +259,500 @@ async def test_explicit_mantle_failure_does_not_probe_runtime() -> None: assert result.status_code == 404 assert result.tokenizer_type == "bedrock_mantle_api" assert bedrock.posted_hosts() == (MANTLE_HOST,) + + +def _invoke_body(request: httpx.Request) -> dict[str, JsonValue]: + invoke_model: Final = _JSON_BODY.validate_json(request.content)["input"] + assert isinstance(invoke_model, dict) + encoded: Final = invoke_model["invokeModel"] + assert isinstance(encoded, dict) and isinstance(encoded["body"], str) + return _JSON_BODY.validate_json(base64.b64decode(encoded["body"])) + + +_CLAUDE_REPLY: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} +_CONTEXT_EDITS: Final[dict[str, JsonValue]] = { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] +} +_THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = { + "thinking": {"type": "enabled", "budget_tokens": 2000}, + "tool_choice": {"type": "auto"}, +} +_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}} + + +_CITY_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}} + + +_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS} + + +async def _messages_inference_and_count( + deployment_model: str, + model_to_use: str, + request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS, + tool: Mapping[str, JsonValue] = _WEATHER_TOOL, + runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}), + inference_reply: Mapping[str, object] = _CLAUDE_REPLY, +) -> tuple[_Bedrock, _Bedrock]: + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, inference_reply), mantle=(200, inference_reply)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + tools=[dict(tool)], + max_tokens=4096, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + **request_options, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177})) + result: Final = await _counter(bedrock).count_tokens( + model_to_use=model_to_use, + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}}, + request_model="claude", + tools=[dict(tool)], + request_options=request_options, + ) + assert result is not None + assert result.error is False + return inference, bedrock + + +def _betas_besides_effort(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]: + betas: Final = body.get("anthropic_beta") or [] + assert isinstance(betas, list) + return tuple(beta for beta in betas if beta != ANTHROPIC_EFFORT_BETA_HEADER) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "model_to_use"), + ( + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-sonnet-4-6"), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"), + ), +) +async def test_runtime_count_sends_the_options_and_context_edits_invoke_inference_sends( + deployment_model: str, model_to_use: str +) -> None: + inference, bedrock = await _messages_inference_and_count(deployment_model, model_to_use) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _invoke_body(bedrock.requests[0]) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + fields: Final = ("thinking", "tool_choice", "output_config", "context_management") + assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} + assert _betas_besides_effort(counted) == _betas_besides_effort(sent) + thinking: Final = counted["thinking"] + assert isinstance(thinking, dict) + budget_tokens: Final = thinking.get("budget_tokens", 0) + assert isinstance(budget_tokens, int) and isinstance(counted["max_tokens"], int) + assert counted["max_tokens"] > budget_tokens + + +@pytest.mark.asyncio +async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5" + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + fields: Final = ("thinking", "tool_choice", "output_config", "context_management") + assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} + assert "anthropic_beta" not in counted + assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"] + + +_DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}} + + +def _sent_betas(request: httpx.Request) -> tuple[str, ...]: + return tuple(sorted(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "request_options", "expected_beta"), + ( + ( + "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", + {"output_config": {"effort": "low"}}, + ANTHROPIC_EFFORT_BETA_HEADER, + ), + ("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER), + ), + ids=("effort", "thinking_display"), +) +async def test_runtime_count_sends_the_betas_invoke_inference_sends( + deployment_model: str, request_options: dict[str, JsonValue], expected_beta: str +) -> None: + inference, bedrock = await _messages_inference_and_count( + deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta") + assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta") + assert isinstance(counted_betas, list) and expected_beta in counted_betas + + +_EFFORT: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}} +_CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = { + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]} +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_options", "expected_betas"), + ((_DISPLAY_THINKING, (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)), (_EFFORT, ())), + ids=("thinking_display", "effort"), +) +async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends( + request_options: dict[str, JsonValue], expected_betas: tuple[str, ...] +) -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/invoke/us.anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + request_options, + runtime=(400, UNSUPPORTED), + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) + sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", []) + assert isinstance(sent_betas, list) + assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) == expected_betas + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "request_options"), + ( + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _EFFORT), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLEAR_THINKING_EDITS), + ("bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", _EFFORT), + ), + ids=("effort", "clear_thinking_sets_effort", "inference_sends_the_effort_beta"), +) +async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_betas_invoke_inference_sends( + deployment_model: str, request_options: dict[str, JsonValue] +) -> None: + inference, bedrock = await _messages_inference_and_count( + deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _invoke_body(bedrock.requests[0]) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + counted_config: Final = counted.get("output_config") + assert isinstance(counted_config, dict) and counted_config.get("effort") == "low" + counted_betas: Final = counted.get("anthropic_beta") + sent_betas: Final = sent.get("anthropic_beta", []) + assert isinstance(counted_betas, list) and isinstance(sent_betas, list) + assert counted_betas.count(ANTHROPIC_EFFORT_BETA_HEADER) == 1 + assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} + + +@pytest.mark.asyncio +async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/invoke/us.anthropic.claude-sonnet-4-6", + "us.anthropic.claude-sonnet-4-6", + {**_CITY_FORMAT, "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _invoke_body(bedrock.requests[0]) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + assert counted.get("output_config") == sent.get("output_config") + assert counted.get("anthropic_beta") == sent.get("anthropic_beta") + + +_CONVERSE_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4}, + "metrics": {"latencyMs": 1}, +} +_NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = { + "tool_choice": {"type": "auto", "disable_parallel_tool_use": True} +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "inference_reply"), + ( + ("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY), + ), + ids=("converse", "invoke"), +) +async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it( + deployment_model: str, inference_reply: Mapping[str, object] +) -> None: + inference, bedrock = await _messages_inference_and_count( + deployment_model, + deployment_model.split("/")[-1], + _NO_PARALLEL_TOOL_USE, + inference_reply=inference_reply, + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted_tool_choice: Final = _invoke_body(bedrock.requests[0]).get("tool_choice") + assert isinstance(counted_tool_choice, dict) and counted_tool_choice.get("type") == "auto" + assert ("disable_parallel_tool_use" in counted_tool_choice) == ( + b"disable_parallel_tool_use" in inference.requests[0].content + ) + + +@pytest.mark.asyncio +async def test_count_on_a_stand_in_model_counts_the_messages_and_output_config_the_deployment_model_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/us.anthropic.claude-haiku-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _CITY_FORMAT + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + fields: Final = ("messages", "output_config") + assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} + + +@pytest.mark.asyncio +async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + assert _sent_betas(bedrock.requests[0]) == _sent_betas(inference.requests[0]) + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0]) + + +def _mantle_answers(status_code: int, body: Mapping[str, JsonValue]) -> Callable[[httpx.Request], httpx.Response]: + return lambda request: httpx.Response(status_code, json=dict(body), request=request) + + +def _mantle_unreachable(error: type[httpx.TransportError]) -> Callable[[httpx.Request], httpx.Response]: + def unreachable(request: httpx.Request) -> httpx.Response: + raise error("bedrock-mantle is unreachable", request=request) + + return unreachable + + +_MANTLE_REJECTION: Final[dict[str, JsonValue]] = { + "type": "error", + "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}, +} +_MANTLE_FAILURE: Final[dict[str, JsonValue]] = {"type": "error", "error": {"type": "api_error", "message": "down"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_options", "mantle", "expected"), + ( + (_DISPLAY_THINKING, _mantle_unreachable(httpx.ConnectError), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_unreachable(httpx.ReadTimeout), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(503, _MANTLE_FAILURE), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(429, _MANTLE_FAILURE), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(403, _MANTLE_FAILURE), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(400, _MANTLE_REJECTION), (True, 400, "bedrock_mantle_api")), + (_DISPLAY_THINKING, _mantle_answers(422, _MANTLE_REJECTION), (True, 422, "bedrock_mantle_api")), + (_DISPLAY_THINKING, _mantle_answers(200, {"input_tokens": 2177}), (False, None, "bedrock_mantle_api")), + ({}, _mantle_answers(503, _MANTLE_FAILURE), (True, 503, "bedrock_mantle_api")), + ), + ids=( + "mantle-unreachable", + "mantle-timeout", + "mantle-5xx", + "mantle-rate-limit", + "mantle-denied", + "mantle-rejects-the-options", + "mantle-rejects-the-body", + "mantle-counts", + "no-options-mantle-5xx", + ), +) +async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or_rejects_the_request( + request_options: dict[str, JsonValue], + mantle: Callable[[httpx.Request], httpx.Response], + expected: tuple[bool, int | None, str], +) -> None: + def bedrock(request: httpx.Request) -> httpx.Response: + if request.url.host == RUNTIME_HOST: + return httpx.Response(400, json=UNSUPPORTED, request=request) + return mantle(request) + + result: Final = await BedrockTokenCounter( + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)) + ).count_tokens( + model_to_use="us.anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment={ + "litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"} + }, + request_model="claude-opus-5-5", + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.status_code, result.tokenizer_type) == expected + + +_CALLER_BETAS: Final = (ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value, "context-1m-2025-08-07") + + +@pytest.mark.asyncio +async def test_runtime_count_sends_the_caller_betas_invoke_inference_keeps_for_the_model() -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0" + model_info: Final[dict[str, JsonValue]] = {"supports_per_turn_control": True} + thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}} + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + max_tokens=4096, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + extra_headers={"anthropic-beta": ",".join(_CALLER_BETAS)}, + model_info=model_info, + **thinking, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + result: Final = await _counter(bedrock).count_tokens( + model_to_use=deployment_model.removeprefix("bedrock/invoke/"), + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info}, + request_model="claude", + request_options=thinking, + anthropic_beta=",".join(_CALLER_BETAS), + ) + + assert result is not None + assert result.error is False + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta") + assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta") + assert isinstance(counted_betas, list) and set(_CALLER_BETAS) <= set(counted_betas) + + +@pytest.mark.asyncio +async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/invoke/us.anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + _THINKING_AND_TOOL_CHOICE, + tool={**_WEATHER_TOOL, "eager_input_streaming": True}, + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] + + +def _texts(message: JsonValue) -> tuple[str, ...]: + content: Final = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + return () + return tuple(text for block in content if isinstance(block, dict) and isinstance(text := block.get("text"), str)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("native_output", "request_options", "schema_inlined"), + ( + (False, _CITY_FORMAT, True), + (True, _CITY_FORMAT, False), + (False, _THINKING_AND_TOOL_CHOICE, False), + ), + ids=("inlined_schema", "native_schema", "no_format"), +) +async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends( + native_output: bool, request_options: dict[str, JsonValue], schema_inlined: bool +) -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-count-probe-v1:0" + model_info: Final = {"supports_bedrock_native_structured_output": native_output} + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + max_tokens=4096, + model_info=model_info, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + **request_options, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-count-probe-v1:0", + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info}, + request_model="claude", + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 412) + counted: Final = _invoke_body(bedrock.requests[0])["messages"] + assert counted == _JSON_BODY.validate_json(inference.requests[0].content)["messages"] + assert isinstance(counted, list) + assert (json.dumps(_CITY_SCHEMA) in _texts(counted[-1])) is schema_inlined + assert (counted == MESSAGES) is not schema_inlined + + +_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("messages", "tools", "request_options"), + ( + (MESSAGES, None, {"output_config": {"effort": "fastest"}}), + (MESSAGES, [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], _THINKING), + (MESSAGES, ["get_weather"], _THINKING), + ([{"role": "user", "content": 5}], None, _THINKING), + ([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING), + ), + ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"), +) +async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock( + messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue] +) -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=messages, + contents=None, + deployment=DEPLOYMENT, + request_model="claude-sonnet-4-5", + tools=tools, + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert bedrock.posted_hosts() == () diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 20dba332c08..6b450c745b3 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1,7 +1,9 @@ import copy import time -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from contextlib import contextmanager +from queue import SimpleQueue +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, patch @@ -28,6 +30,7 @@ from litellm.llms.vertex_ai.common_utils import ( vertex_request_labels_from_litellm_params, ) from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler +from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat @@ -2291,3 +2294,179 @@ async def test_vertex_ai_token_counter_routes_like_generate_content( ) assert (None if result is None else result.total_tokens) == vertex_counted_tokens + + +_PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, +] +_PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, +} + + +async def _partner_counted_bodies( + model: str, request_options: Mapping[str, JsonValue] = MappingProxyType({}) +) -> tuple[dict[str, object], ...]: + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler + + class FakeResponse: + status_code = 200 + + def json(self) -> dict[str, int]: + return {"input_tokens": 37} + + class FakeHttpClient: + posted_bodies: tuple[dict[str, object], ...] = () + + async def post( + self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float + ) -> FakeResponse: + self.posted_bodies = (*self.posted_bodies, json) + return FakeResponse() + + fake_http_client: Final = FakeHttpClient() + with ( + patch.object(handler, "get_async_httpx_client", return_value=fake_http_client), + patch.object( + handler.VertexAIPartnerModelsTokenCounter, + "_ensure_access_token_async", + new=AsyncMock(return_value=("fake-token", "test-project")), + ), + ): + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use=model, + messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS), + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + tools=[_PARTNER_RESPONSES_TOOL], + request_options=request_options, + ) + + assert result is not None + assert result.total_tokens == 37 + return fake_http_client.posted_bodies + + +_PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} + + +async def _vertex_claude_inference_and_count( + tools: list[dict[str, JsonValue]], request_options: Mapping[str, JsonValue] +) -> tuple[httpx.Request, httpx.Request]: + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}] + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def vertex(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response( + 200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY + ) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex)) + with ( + patch.object(handler, "get_async_httpx_client", return_value=client), + patch.object( + VertexBase, "_ensure_access_token_async", new=AsyncMock(return_value=("fake-token", "test-project")) + ), + patch.object(VertexBase, "_ensure_access_token", return_value=("fake-token", "test-project")), + ): + await litellm.anthropic_messages( + model="vertex_ai/claude-haiku-5-5", + messages=messages, + tools=tools, + max_tokens=4096, + vertex_project="test-project", + vertex_location="us-east5", + client=client, + **request_options, + ) + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="claude-haiku-5-5", + messages=messages, + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + tools=tools, + request_options=request_options, + ) + + assert result is not None + assert result.total_tokens == 37 + inference, count = tuple(sent.get() for _ in range(sent.qsize())) + assert count.url.path.endswith("count-tokens:rawPredict") + return inference, count + + +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None: + inference, count = await _vertex_claude_inference_and_count( + [{"name": "get_weather", "input_schema": {"type": "object"}}], + { + "thinking": {"type": "enabled", "budget_tokens": 2000}, + "tool_choice": {"type": "auto"}, + "context_management": { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] + }, + }, + ) + + counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content) + inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content) + fields: Final = ("thinking", "tool_choice", "output_config", "context_management") + assert {name: counted.get(name) for name in fields} == {name: inferred.get(name) for name in fields} + assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"] + + +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_the_tool_search_beta_vertex_messages_inference_sends() -> None: + inference, count = await _vertex_claude_inference_and_count( + [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True}, + ], + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + ) + + assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"] + assert get_tool_search_beta_header("vertex_ai") in count.headers["anthropic-beta"].split(",") + + +@pytest.mark.asyncio +async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="claude-haiku-4-5", + messages=[{"role": "user", "content": "Weather in SF?"}], + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + request_options={"output_config": {"effort": "fastest"}}, + ) + + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + + +@pytest.mark.asyncio +async def test_vertex_llama_count_leaves_out_the_claude_only_options() -> None: + model: Final = "meta/llama-4-maverick-17b-128e-instruct-maas" + + assert await _partner_counted_bodies( + model, request_options={"thinking": {"type": "enabled", "budget_tokens": 2000}} + ) == await _partner_counted_bodies(model) diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index d17fad7e9b3..0ede4f059ab 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import anyio import httpx import pytest +import respx import tiktoken from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, JsonValue @@ -36,6 +37,7 @@ load_dotenv() from fastapi import Request from fastapi.responses import JSONResponse +from starlette.types import Message import litellm from litellm import Router @@ -53,10 +55,11 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( kimi_k3_tokens, load_kimi_k3_encoding, ) +from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensInputType from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter -from litellm.proxy._types import ProxyException, TokenCountRequest +from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.endpoints import ( count_tokens as anthropic_count_tokens, ) @@ -76,6 +79,8 @@ from litellm.proxy.proxy_server import ( token_counter, ) from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.router import CustomRoutingStrategyBase from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from litellm.types.utils import SelectTokenizerResponse, TokenCountMessagesFormat, TokenCountResponse @@ -240,6 +245,9 @@ async def test_anthropic_messages_count_tokens_endpoint(): output_format=None, context_management=None, request_litellm_params=None, + request_options=None, + anthropic_beta=None, + forward_client_headers=False, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -320,6 +328,9 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): output_format=None, context_management=None, request_litellm_params=None, + request_options=None, + anthropic_beta=None, + forward_client_headers=False, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -809,6 +820,7 @@ class _FailingRuntimeHandler(BedrockCountTokensHandler): litellm_params: dict[str, object], resolved_model: str, client: AsyncHTTPHandler | None = None, + input_type: BedrockCountTokensInputType | None = None, ) -> dict[str, JsonValue]: raise self._error @@ -1138,6 +1150,9 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): output_format=None, context_management=None, request_litellm_params=None, + request_options=None, + anthropic_beta=None, + forward_client_headers=False, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1195,6 +1210,9 @@ async def test_anthropic_endpoint_403_permission_error_format(): output_format=None, context_management=None, request_litellm_params=None, + request_options=None, + anthropic_beta=None, + forward_client_headers=False, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1252,6 +1270,9 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): output_format=None, context_management=None, request_litellm_params=None, + request_options=None, + anthropic_beta=None, + forward_client_headers=False, ): raise ProxyException( message="Rate limit exceeded", @@ -1276,6 +1297,660 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = original_token_counter +def _json_request( + body: Mapping[str, object], user_agent: str, headers: Mapping[str, str] = MappingProxyType({}) +) -> Request: + payload: Final = json.dumps(body).encode() + + async def receive() -> Message: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/messages/count_tokens", + "query_string": b"", + "headers": [ + (b"content-type", b"application/json"), + (b"user-agent", user_agent.encode()), + *((name.encode(), value.encode()) for name, value in headers.items()), + ], + }, + receive, + ) + + +_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = { + "thinking": {"type": "adaptive"}, + "tool_choice": {"type": "auto"}, + "output_config": { + "effort": "low", + "format": { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + }, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, +} + + +def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]: + output_config: Final = body.get("output_config") + return frozenset(name for name in _CLAUDE_COUNT_OPTIONS if name in body) | frozenset( + f"output_config.{key}" for key in (output_config if isinstance(output_config, dict) else {}) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("additional_drop_params", "request_params", "expected_paths"), + ( + ( + None, + {}, + { + "thinking", + "tool_choice", + "output_config", + "output_config.effort", + "output_config.format", + "context_management", + }, + ), + ( + ["tool_choice", "output_config.format"], + {}, + {"thinking", "output_config", "output_config.effort", "context_management"}, + ), + ( + ["context_management"], + {}, + {"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"}, + ), + ( + None, + {"additional_drop_params": ["thinking", "context_management"]}, + {"tool_choice", "output_config", "output_config.effort", "output_config.format"}, + ), + ), + ids=( + "nothing-dropped", + "deployment-drops-tool-choice-and-the-format", + "deployment-drops-context-management", + "request-drops-thinking-and-context-management", + ), +) +async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends( + monkeypatch: pytest.MonkeyPatch, + additional_drop_params: list[str] | None, + request_params: dict[str, JsonValue], + expected_paths: set[str], +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "claude-agent", + "litellm_params": { + "model": "anthropic/claude-haiku-5-5", + "api_key": "sk-ant-test", + "additional_drop_params": additional_drop_params, + }, + } + ] + ) + request: Final = _json_request( + { + "model": "claude-agent", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + **_CLAUDE_COUNT_OPTIONS, + **request_params, + }, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 7 + sent_request: Final = route.calls.last.request + assert _sent_option_paths(json.loads(sent_request.content)) == expected_paths + assert ("context-management-2025-06-27" in sent_request.headers["anthropic-beta"]) == ( + "context_management" in expected_paths + ) + + +_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value + + +def _claude_router(deployment_model: str, deployment_params: Mapping[str, JsonValue]) -> Router: + return Router( + model_list=[ + { + "model_name": "claude-agent", + "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test", **deployment_params}, + } + ] + ) + + +async def _count_on_a_claude_deployment( + monkeypatch: pytest.MonkeyPatch, + deployment_model: str, + body: Mapping[str, JsonValue], + reply: Callable[[httpx.Request], httpx.Response], + user_agent: str = "python-httpx/0.28.1", + deployment_params: Mapping[str, JsonValue] = MappingProxyType({}), +) -> tuple[object, tuple[httpx.Request, ...]]: + import litellm.proxy.proxy_server as proxy_server + + router: Final = _claude_router(deployment_model, deployment_params) + request: Final = _json_request( + {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, + user_agent, + {"anthropic-beta": _CALLER_BETA}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(side_effect=reply) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + return response, tuple(call.request for call in route.calls) + + +def _counts_only_without_options(request: httpx.Request) -> httpx.Response: + if _sent_option_paths(json.loads(request.content)): + return httpx.Response( + 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}} + ) + return httpx.Response(200, json={"input_tokens": 7}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "options", "rejected_by_anthropic"), + ( + ("anthropic/claude-haiku-4-5", {"output_config": {"effort": "fastest"}}, False), + ("anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, True), + ), + ids=("rejected-by-the-count-transform", "rejected-by-anthropic"), +) +async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_claude_count_rejects_them( + monkeypatch: pytest.MonkeyPatch, + deployment_model: str, + options: dict[str, JsonValue], + rejected_by_anthropic: bool, +) -> None: + response, requests = await _count_on_a_claude_deployment( + monkeypatch, deployment_model, options, _counts_only_without_options + ) + + assert not isinstance(response, JSONResponse) + assert response["input_tokens"] == 7 + assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [ + *((True,) if rejected_by_anthropic else ()), + False, + ] + assert [_CALLER_BETA in sent.headers.get("anthropic-beta", "") for sent in requests] == [ + *((True,) if rejected_by_anthropic else ()), + False, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("status_code", "counted_again"), ((401, False), (403, False), (404, True), (422, True))) +async def test_anthropic_count_tokens_counts_again_only_when_the_claude_count_rejects_the_request( + monkeypatch: pytest.MonkeyPatch, status_code: int, counted_again: bool +) -> None: + response, requests = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-haiku-5-5", + _CLAUDE_COUNT_OPTIONS, + lambda _: httpx.Response(status_code, json={"type": "error", "error": {"type": "error", "message": "denied"}}), + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == status_code + assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [ + True, + *((False,) if counted_again else ()), + ] + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_warns_with_every_option_it_counts_again_without( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + verbose_proxy_logger.addHandler(caplog.handler) + try: + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _count_on_a_claude_deployment( + monkeypatch, "anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, _counts_only_without_options + ) + finally: + verbose_proxy_logger.removeHandler(caplog.handler) + + warnings: Final = { + record.getMessage() for record in caplog.records if "counts again without" in record.getMessage() + } + (warning,) = warnings + assert all(option in warning for option in _CLAUDE_COUNT_OPTIONS) + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, requests = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-haiku-5-5", + _CLAUDE_COUNT_OPTIONS, + lambda _: httpx.Response(429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow"}}), + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 429 + assert len(requests) == 1 + + +_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" +_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" +_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = { + "model": "bedrock/us.anthropic.claude-count-probe-v1:0", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-east-1", +} + + +async def _count_on_a_bedrock_claude_deployment( + monkeypatch: pytest.MonkeyPatch, + body: Mapping[str, JsonValue], + runtime: Callable[[httpx.Request], httpx.Response], + mantle: Callable[[httpx.Request], httpx.Response], +) -> tuple[object, tuple[httpx.Request, ...]]: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}]) + request: Final = _json_request( + {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock as bedrock: + bedrock.route(host=_BEDROCK_RUNTIME_HOST).mock(side_effect=runtime) + bedrock.route(host=_BEDROCK_MANTLE_HOST).mock(side_effect=mantle) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + return response, tuple(call.request for call in bedrock.calls) + + +def _runtime_counts_only_without_options(request: httpx.Request) -> httpx.Response: + if "invokeModel" in json.loads(request.content)["input"]: + return httpx.Response(400, json={"message": "thinking: unsupported"}) + return httpx.Response(200, json={"inputTokens": 9}) + + +def _mantle_unreachable(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("bedrock-mantle is unreachable", request=request) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "malformed", + ( + {"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}]}, + {"messages": [{"role": "user", "content": 5}]}, + {"messages": [{"role": "user", "content": [{"text": "Weather in SF?"}]}]}, + ), + ids=("tool_type_not_a_string", "content_not_iterable", "block_without_type"), +) +async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_request_again_without_the_options( + monkeypatch: pytest.MonkeyPatch, malformed: dict[str, JsonValue] +) -> None: + response, requests = await _count_on_a_bedrock_claude_deployment( + monkeypatch, + {**malformed, "thinking": {"type": "enabled", "budget_tokens": 2000}}, + _runtime_counts_only_without_options, + _mantle_unreachable, + ) + + assert not isinstance(response, JSONResponse) + assert response["input_tokens"] == 9 + assert [request.url.host for request in requests] == [_BEDROCK_RUNTIME_HOST] + assert "invokeModel" not in json.loads(requests[0].content)["input"] + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_options_when_mantle_is_unreachable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, requests = await _count_on_a_bedrock_claude_deployment( + monkeypatch, + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + _runtime_counts_only_without_options, + _mantle_unreachable, + ) + + assert not isinstance(response, JSONResponse) + assert response["input_tokens"] == 9 + hosts: Final = [request.url.host for request in requests] + assert hosts[0] == hosts[-1] == _BEDROCK_RUNTIME_HOST + assert _BEDROCK_MANTLE_HOST in hosts + assert "invokeModel" not in json.loads(requests[-1].content)["input"] + + +_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management" +_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management" +_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} +_CLAUDE_MESSAGE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} + + +class _RoutesTo(CustomRoutingStrategyBase): + def __init__(self, router: Router, api_key: str) -> None: + self._router: Final = router + self._api_key: Final = api_key + + async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]: + return next( + deployment + for deployment in self._router.model_list + if deployment["litellm_params"]["api_key"] == self._api_key + ) + + +def _mixed_drop_group(routed_key: str) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "claude-agent", + "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": _KEEPS_CONTEXT_MANAGEMENT}, + }, + { + "model_name": "claude-agent", + "litellm_params": { + "model": "anthropic/claude-haiku-5-5", + "api_key": _DROPS_CONTEXT_MANAGEMENT, + "additional_drop_params": ["context_management"], + }, + }, + ] + ) + router.set_custom_routing_strategy(_RoutesTo(router, routed_key)) + return router + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routed_key", (_KEEPS_CONTEXT_MANAGEMENT, _DROPS_CONTEXT_MANAGEMENT)) +async def test_anthropic_count_tokens_in_a_mixed_drop_group_sends_context_management_when_the_routed_inference_does( + monkeypatch: pytest.MonkeyPatch, routed_key: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key)) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + messages: Final = [{"role": "user", "content": "Weather in SF?"}] + + with respx.mock: + inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + count_route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + await _mixed_drop_group(routed_key).aanthropic_messages( + model="claude-agent", messages=messages, max_tokens=16, context_management=_CONTEXT_EDITS + ) + response: Final = await anthropic_count_tokens( + _json_request( + {"model": "claude-agent", "messages": messages, "context_management": _CONTEXT_EDITS}, + "python-httpx/0.28.1", + ), + UserAPIKeyAuth(), + ) + + assert not isinstance(response, JSONResponse) + inference: Final = inference_route.calls.last.request + count: Final = count_route.calls.last.request + assert inference.headers["x-api-key"] == count.headers["x-api-key"] == routed_key + assert ("context_management" in json.loads(count.content)) == ( + "context_management" in json.loads(inference.content) + ) + assert ("context_management" in json.loads(count.content)) == (routed_key == _KEEPS_CONTEXT_MANAGEMENT) + + +_AZURE_AI_CLAUDE: Final[dict[str, str]] = { + "model": "azure_ai/claude-haiku-5-5", + "api_base": "https://example-resource.services.ai.azure.com/anthropic", + "api_key": "azure-test-key", +} +_ANTHROPIC_CLAUDE: Final[dict[str, str]] = {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-ant-test"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("litellm_params", "general_settings", "forwarded_model_groups", "caller_beta_sent"), + ( + (_ANTHROPIC_CLAUDE, {}, None, True), + (_AZURE_AI_CLAUDE, {}, None, False), + (_AZURE_AI_CLAUDE, {"forward_client_headers_to_llm_api": True}, None, True), + (_AZURE_AI_CLAUDE, {}, ["claude-agent"], True), + ), + ids=( + "anthropic-api-header-provider", + "other-provider", + "other-provider-forwarding-client-headers", + "other-provider-in-a-forwarding-model-group", + ), +) +async def test_anthropic_count_tokens_sends_the_caller_beta_only_when_inference_does( + monkeypatch: pytest.MonkeyPatch, + litellm_params: dict[str, str], + general_settings: dict[str, object], + forwarded_model_groups: list[str] | None, + caller_beta_sent: bool, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.types.router import ModelGroupSettings + + router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": litellm_params}]) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr( + litellm, "model_group_settings", ModelGroupSettings(forward_client_headers_to_llm_api=forwarded_model_groups) + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + body: Final[dict[str, JsonValue]] = { + "model": "claude-agent", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "thinking": {"type": "enabled", "budget_tokens": 2000}, + } + caller_headers: Final = {"anthropic-beta": _CALLER_BETA} + proxy_data: Final = await add_litellm_data_to_request( + data=dict(body), + request=_json_request(body, "python-httpx/0.28.1", caller_headers), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"), + proxy_config=proxy_server.ProxyConfig(), + general_settings=general_settings, + ) + + with respx.mock: + inference_route: Final = respx.post(url__regex=r".*/v1/messages$").mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + count_route: Final = respx.post(url__regex=r".*/v1/messages/count_tokens$").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + await router.aanthropic_messages( + **body, + max_tokens=4096, + **{name: proxy_data[name] for name in ("provider_specific_header", "headers") if name in proxy_data}, + ) + response: Final = await anthropic_count_tokens( + _json_request(body, "python-httpx/0.28.1", caller_headers), UserAPIKeyAuth(api_key="sk-1234") + ) + + assert not isinstance(response, JSONResponse) + inference_betas: Final = inference_route.calls.last.request.headers.get("anthropic-beta", "").split(",") + count_betas: Final = count_route.calls.last.request.headers.get("anthropic-beta", "").split(",") + assert (_CALLER_BETA in inference_betas) == (_CALLER_BETA in count_betas) == caller_beta_sent + + +async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + ) + ) + await litellm.anthropic_messages( + model=f"anthropic/{model}", + messages=[{"role": "user", "content": "Weather in SF?"}], + max_tokens=4096, + api_key="sk-ant-test", + **options, + ) + return json.loads(route.calls.last.request.content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_drop_params", "user_agent"), + ( + ({"drop_params": True}, "python-httpx/0.28.1"), + ({}, "claude-cli/2.1.0 (external, cli)"), + ), +) +async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send( + monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"} + dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True}) + kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking}) + assert dropped["thinking"] != kept["thinking"] + + response, (count,) = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-opus-4-7", + {"thinking": thinking, **request_drop_params}, + lambda _: httpx.Response(200, json={"input_tokens": 7}), + user_agent, + ) + + assert not isinstance(response, JSONResponse) + counted: Final = json.loads(count.content) + assert {name: counted.get(name) for name in ("thinking", "output_config")} == { + name: dropped.get(name) for name in ("thinking", "output_config") + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_drop_params", "request_params"), + ( + (True, {"drop_params": "maybe"}), + (True, {"drop_params": False}), + (True, {"drop_params": None}), + (True, {}), + (None, {"drop_params": " true "}), + ), +) +async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_deployments_like_inference( + monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_params: Mapping[str, JsonValue] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"} + deployment_params: Final[Mapping[str, JsonValue]] = ( + {} if deployment_drop_params is None else {"drop_params": deployment_drop_params} + ) + with respx.mock: + inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + await _claude_router("anthropic/claude-opus-5-5", deployment_params).aanthropic_messages( + model="claude-agent", + messages=[{"role": "user", "content": "Weather in SF?"}], + max_tokens=4096, + thinking=thinking, + **request_params, + ) + sent: Final = json.loads(inference_route.calls.last.request.content) + + response, (count,) = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-opus-5-5", + {"thinking": thinking, **request_params}, + lambda _: httpx.Response(200, json={"input_tokens": 7}), + deployment_params=deployment_params, + ) + + assert not isinstance(response, JSONResponse) + assert json.loads(count.content).get("thinking") == sent.get("thinking") + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + output_format: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + + response, (count,) = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-haiku-5-5", + {"output_format": output_format}, + lambda _: httpx.Response(200, json={"input_tokens": 7}), + ) + + assert not isinstance(response, JSONResponse) + counted: Final = json.loads(count.content) + assert "output_format" not in counted + assert counted["output_config"]["format"] == output_format + + class _ScriptedCounter(BaseTokenCounter): def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: self._replies = replies @@ -1299,6 +1974,8 @@ class _ScriptedCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: object = None, + request_options: object = None, + anthropic_beta: object = None, ) -> TokenCountResponse | None: self.calls.append((model_to_use, messages, tools, system)) self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name")) @@ -3037,7 +3714,7 @@ class _PartnerCountRequest(BaseModel): async def _vertex_partner_word_count( - model: str, request_data: JsonValue, litellm_params: JsonValue + model: str, request_data: JsonValue, litellm_params: JsonValue, betas: object = () ) -> dict[str, JsonValue]: body: Final = _PartnerCountRequest.model_validate(request_data) return {"input_tokens": sum(len(message.content.split()) for message in body.messages)}