mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): send inference's betas and tools on the Claude count, and retry a rejected count without the forwarded options
The Claude count now builds its anthropic-beta set and tools through the same provider config inference uses, so effort, thinking display, the caller's anthropic-beta header, and Bedrock's tool rewrites match the request inference would send. When the provider still rejects a count that carries the forwarded thinking, tool, output, or context options with a 4xx other than 429, the proxy counts once more without them, which is the request it sent before those options were forwarded. Pass-through Bedrock counts go back to detecting Converse or InvokeModel from the body alone
This commit is contained in:
parent
8db6f0d5ed
commit
f84c244f3e
23 changed files with 654 additions and 177 deletions
|
|
@ -51,6 +51,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
|
|
@ -96,7 +97,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
pass
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, context_management, deployment
|
||||
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, anthropic_beta, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "anthropic_api")
|
||||
|
|
@ -138,7 +139,9 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
else [dict(message) for message in claude_options.messages],
|
||||
auth_header=auth_header,
|
||||
api_base=api_base,
|
||||
tools=tools,
|
||||
tools=tools
|
||||
if claude_options.messages is None
|
||||
else [dict(tool) for tool in claude_options.tools or ()] or None,
|
||||
system=system,
|
||||
optional_params={**count_options, **claude_options.options},
|
||||
betas=claude_options.betas,
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm.litellm_core_utils.url_utils import join_api_base
|
|||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
|
||||
|
|
@ -337,31 +337,41 @@ class ClaudeCountOptions:
|
|||
options: Mapping[str, JsonValue]
|
||||
betas: tuple[str, ...]
|
||||
messages: tuple[Mapping[str, JsonValue], ...] | None
|
||||
tools: tuple[Mapping[str, JsonValue], ...] | None
|
||||
|
||||
|
||||
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None)
|
||||
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
|
||||
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
|
||||
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
|
||||
|
||||
|
||||
def _with_provider_betas(headers: Mapping[str, str], provider: str) -> Mapping[str, str]:
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
|
||||
beta_header: Final = headers.get("anthropic-beta")
|
||||
if beta_header is None:
|
||||
return headers
|
||||
supported: Final = filter_and_transform_beta_headers(
|
||||
[beta.strip() for beta in beta_header.split(",") if beta.strip()], provider
|
||||
)
|
||||
other_headers: Final = {name: value for name, value in headers.items() if name != "anthropic-beta"}
|
||||
return {**other_headers, "anthropic-beta": ",".join(supported)} if supported else other_headers
|
||||
|
||||
|
||||
def claude_count_options(
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
request_options: Mapping[str, JsonValue],
|
||||
context_management: CountContextManagement | None,
|
||||
anthropic_beta: str | None,
|
||||
deployment: Mapping[str, object] | None,
|
||||
) -> ClaudeCountOptions | ChatCountTranslationError:
|
||||
sent: Final = {
|
||||
**{name: value for name, value in request_options.items() if value is not None},
|
||||
**({} if context_management is None else {"context_management": context_management.spec}),
|
||||
}
|
||||
sent: Final = {name: value for name, value in request_options.items() if value is not None}
|
||||
if not sent:
|
||||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, context_management_betas
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
|
@ -372,37 +382,39 @@ def claude_count_options(
|
|||
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
leading_system, conversation = split_leading_system_run(messages)
|
||||
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
||||
sent_tools: Final = copy.deepcopy([dict(tool) for tool in tools or ()])
|
||||
sent_params: Final = {
|
||||
**copy.deepcopy(sent),
|
||||
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
|
||||
}
|
||||
beta_headers: Final = messages_config.headers_with_anthropic_betas(
|
||||
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
|
||||
)
|
||||
sent_headers: Final = dict(
|
||||
_with_provider_betas(beta_headers, provider.value)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else beta_headers
|
||||
)
|
||||
try:
|
||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||
model=model,
|
||||
messages=sent_conversation,
|
||||
anthropic_messages_optional_request_params={
|
||||
**copy.deepcopy(sent),
|
||||
**({"tools": sent_tools} if sent_tools else {}),
|
||||
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
|
||||
},
|
||||
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
|
||||
litellm_params=GenericLiteLLMParams.model_validate(
|
||||
{
|
||||
"drop_params": deployment_params.get("drop_params"),
|
||||
"model_info": resolved_deployment.get("model_info"),
|
||||
}
|
||||
),
|
||||
headers={},
|
||||
headers=sent_headers,
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
|
||||
options: Final = _count_options(transformed)
|
||||
betas: Final = sorted(context_management_betas(options.get("context_management")))
|
||||
return ClaudeCountOptions(
|
||||
options=options,
|
||||
betas=tuple(
|
||||
filter_and_transform_beta_headers(betas, provider.value)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else betas
|
||||
),
|
||||
options=_count_options(transformed),
|
||||
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
|
||||
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
|
||||
tools=transformed.get("tools"),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -413,6 +413,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
messages=messages,
|
||||
)
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return self._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared beta step takes and returns bare dicts
|
||||
headers=dict(headers),
|
||||
optional_params=dict(optional_params),
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _translate_reasoning_effort_to_anthropic(
|
||||
model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
|
|
@ -110,7 +111,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
pass
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, context_management, deployment
|
||||
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, anthropic_beta, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
||||
|
|
@ -142,7 +143,9 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
tools=tools,
|
||||
tools=tools
|
||||
if claude_options.messages is None
|
||||
else [dict(tool) for tool in claude_options.tools or ()] or None,
|
||||
system=system,
|
||||
optional_params={**count_options, **claude_options.options},
|
||||
betas=claude_options.betas,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -127,6 +127,18 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
"""
|
||||
return headers, None
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return headers
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
beta_header: Final = next((value for name, value in headers.items() if name.lower() == "anthropic-beta"), "")
|
||||
return tuple(sorted(frozenset(beta.strip() for beta in beta_header.split(",")) - {""}))
|
||||
|
||||
def should_filter_anthropic_beta_headers(self) -> bool:
|
||||
"""
|
||||
Whether ``anthropic-beta`` header values should be filtered down to the
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ class BaseTokenCounter(ABC):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Bedrock Token Counter implementation using the CountTokens API.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
|
@ -117,21 +117,19 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
) -> bool:
|
||||
return custom_llm_provider == LlmProviders.BEDROCK.value
|
||||
|
||||
async def _count_with(
|
||||
self,
|
||||
handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler,
|
||||
tokenizer_type: str,
|
||||
request_data: Mapping[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
resolved_model: str,
|
||||
) -> CountTokensOutcome:
|
||||
def _count_on_mantle(
|
||||
self, request_data: Mapping[str, object], litellm_params: Mapping[str, object], resolved_model: str
|
||||
) -> Awaitable[Mapping[str, JsonValue]]:
|
||||
return self._mantle_handler.handle_count_tokens_request(
|
||||
request_data=dict(request_data),
|
||||
litellm_params=dict(litellm_params),
|
||||
resolved_model=resolved_model,
|
||||
client=self._client,
|
||||
)
|
||||
|
||||
async def _count_with(self, count: Awaitable[Mapping[str, JsonValue]], tokenizer_type: str) -> CountTokensOutcome:
|
||||
try:
|
||||
result: Final = await handler.handle_count_tokens_request(
|
||||
request_data=dict(request_data),
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
client=self._client,
|
||||
)
|
||||
result: Final = await count
|
||||
reply: Final = _CountTokensReply.model_validate(result)
|
||||
except BedrockError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -157,6 +155,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
if not messages:
|
||||
return None
|
||||
|
|
@ -182,14 +181,15 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
if isinstance(translated, ChatCountTranslationError):
|
||||
return translated.response(request_model, model_to_use, primary_tokenizer)
|
||||
sent_messages: Final = messages if translated is None else translated.messages
|
||||
sent_tools: Final = tools if translated is None else translated.tools
|
||||
claude_options: Final = (
|
||||
claude_count_options(
|
||||
counted_model,
|
||||
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
|
||||
sent_messages,
|
||||
tools if translated is None else translated.tools,
|
||||
sent_tools,
|
||||
request_options,
|
||||
context_management,
|
||||
anthropic_beta,
|
||||
deployment,
|
||||
)
|
||||
if is_mantle_claude_model(resolved_model)
|
||||
|
|
@ -197,29 +197,37 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, primary_tokenizer)
|
||||
counted_messages: Final = sent_messages if claude_options.messages is None else claude_options.messages
|
||||
invoke_fields: Final = _invoke_fields(claude_options)
|
||||
runtime_request: Final = {
|
||||
**(
|
||||
_request_data(model_to_use, counted_messages, tools, system)
|
||||
if translated is None
|
||||
else _request_data(model_to_use, counted_messages, translated.tools, translated.system)
|
||||
**_request_data(
|
||||
model_to_use,
|
||||
sent_messages if claude_options.messages is None else claude_options.messages,
|
||||
sent_tools if claude_options.messages is None else claude_options.tools,
|
||||
system if translated is None else translated.system,
|
||||
),
|
||||
**_invoke_fields(claude_options),
|
||||
**invoke_fields,
|
||||
}
|
||||
mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
|
||||
|
||||
primary: Final = (
|
||||
await self._count_with(
|
||||
self._mantle_handler, primary_tokenizer, mantle_request, litellm_params, resolved_model
|
||||
self._count_on_mantle(mantle_request, litellm_params, resolved_model), primary_tokenizer
|
||||
)
|
||||
if use_mantle
|
||||
else await self._count_with(
|
||||
self._runtime_handler, primary_tokenizer, runtime_request, litellm_params, resolved_model
|
||||
self._runtime_handler.handle_count_tokens_request(
|
||||
request_data=dict(runtime_request),
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
client=self._client,
|
||||
input_type="invokeModel" if invoke_fields else None,
|
||||
),
|
||||
primary_tokenizer,
|
||||
)
|
||||
)
|
||||
outcome: Final = (
|
||||
await self._count_with(
|
||||
self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
|
||||
self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE
|
||||
)
|
||||
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
|
||||
else primary
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.bedrock.base_aws_llm import run_aws_signing
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
|
||||
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig, BedrockCountTokensInputType
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
|
||||
|
||||
|
||||
|
|
@ -29,6 +29,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
litellm_params: dict[str, Any],
|
||||
resolved_model: str,
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
input_type: BedrockCountTokensInputType | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Handle a CountTokens request using existing LiteLLM patterns.
|
||||
|
|
@ -43,7 +44,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
"""
|
||||
try:
|
||||
# Validate the request
|
||||
self.validate_count_tokens_request(request_data)
|
||||
self.validate_count_tokens_request(request_data, input_type)
|
||||
|
||||
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
|
||||
|
||||
|
|
@ -58,7 +59,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
verbose_logger.debug("Retrieved AWS region: %s", aws_region_name)
|
||||
|
||||
# Transform request to Bedrock format (supports both Converse and InvokeModel)
|
||||
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data)
|
||||
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(
|
||||
request_data=request_data, input_type=input_type
|
||||
)
|
||||
|
||||
verbose_logger.debug("Transformed request: %s", bedrock_request)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ to AWS Bedrock's CountTokens API format and vice versa.
|
|||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, Literal
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -19,9 +19,7 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
|
|||
# max_tokens field; CountTokens only counts input, so it has no effect
|
||||
# on any generation.
|
||||
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024
|
||||
_INVOKE_MODEL_ONLY_FIELDS: Final = frozenset(
|
||||
("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta")
|
||||
)
|
||||
BedrockCountTokensInputType: TypeAlias = Literal["converse", "invokeModel"]
|
||||
|
||||
|
||||
def _invoke_model_max_tokens(thinking: JsonValue) -> int:
|
||||
|
|
@ -86,7 +84,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
- Response: {"inputTokens": <number>}
|
||||
"""
|
||||
|
||||
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> Literal["converse", "invokeModel"]:
|
||||
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> BedrockCountTokensInputType:
|
||||
"""
|
||||
Detect whether to use 'converse' or 'invokeModel' input format.
|
||||
|
||||
|
|
@ -97,8 +95,6 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
'converse' or 'invokeModel'
|
||||
"""
|
||||
messages: Final = request_data.get("messages")
|
||||
if not _INVOKE_MODEL_ONLY_FIELDS.isdisjoint(request_data):
|
||||
return "invokeModel"
|
||||
if isinstance(messages, list):
|
||||
# Anthropic content blocks carry a "type" key ({"type": "text", ...});
|
||||
# Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
|
||||
|
|
@ -119,6 +115,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
def transform_anthropic_to_bedrock_count_tokens(
|
||||
self,
|
||||
request_data: Mapping[str, JsonValue],
|
||||
input_type: BedrockCountTokensInputType | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
Transform request to Bedrock CountTokens format.
|
||||
|
|
@ -149,9 +146,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
}
|
||||
}
|
||||
"""
|
||||
input_type: Final = self._detect_input_type(request_data)
|
||||
|
||||
if input_type == "converse":
|
||||
if (input_type or self._detect_input_type(request_data)) == "converse":
|
||||
return self._transform_to_converse_format(request_data)
|
||||
else:
|
||||
return self._transform_to_invoke_model_format(request_data)
|
||||
|
|
@ -270,23 +265,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
|
||||
return {"input_tokens": input_tokens}
|
||||
|
||||
def validate_count_tokens_request(self, request_data: Mapping[str, JsonValue]) -> None:
|
||||
def validate_count_tokens_request(
|
||||
self, request_data: Mapping[str, JsonValue], input_type: BedrockCountTokensInputType | None = None
|
||||
) -> None:
|
||||
"""
|
||||
Validate the incoming count tokens request.
|
||||
Supports both Converse and InvokeModel input formats.
|
||||
|
||||
Args:
|
||||
request_data: The request payload
|
||||
|
||||
Raises:
|
||||
ValueError: If the request is invalid
|
||||
"""
|
||||
if not request_data.get("model"):
|
||||
raise ValueError("model parameter is required")
|
||||
|
||||
input_type: Final = self._detect_input_type(request_data)
|
||||
|
||||
if input_type == "converse":
|
||||
if (input_type or self._detect_input_type(request_data)) == "converse":
|
||||
# Validate Converse format (messages-based)
|
||||
messages: Final = request_data.get("messages", [])
|
||||
if not messages:
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ else:
|
|||
|
||||
|
||||
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class AmazonAnthropicClaudeMessagesConfig(
|
||||
|
|
@ -105,6 +106,17 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
|
||||
return True
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return headers
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ())
|
||||
|
||||
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
|
||||
|
||||
def get_error_class(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
|
|||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.llms.bedrock.common_utils import build_mantle_messages_url
|
||||
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
|
|
@ -70,6 +71,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
|
|||
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
|
||||
return False
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return BaseAnthropicMessagesConfig.sent_anthropic_betas(self, headers, request)
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -535,6 +535,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
Count tokens using OpenAI's Responses API /input_tokens endpoint.
|
||||
|
|
|
|||
|
|
@ -1453,6 +1453,7 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
|
@ -1519,7 +1520,7 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
sent_messages,
|
||||
tools,
|
||||
request_options,
|
||||
context_management,
|
||||
anthropic_beta,
|
||||
deployment,
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
|
|
@ -1556,7 +1557,9 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
vertex_location=vertex_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
system=system,
|
||||
tools=tools,
|
||||
tools=tools
|
||||
if claude_options.messages is None
|
||||
else [dict(tool) for tool in claude_options.tools or ()] or None,
|
||||
count_options={**count_options, **claude_options.options},
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence
|
|||
from typing import Final
|
||||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -51,6 +51,11 @@ def _vertex_version(headers: Mapping[str, object], model: str, drop_params: bool
|
|||
)
|
||||
|
||||
|
||||
def _is_web_search(tool: Mapping[str, object]) -> bool:
|
||||
tool_type: Final = tool.get("type")
|
||||
return isinstance(tool_type, str) and tool_type.startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value)
|
||||
|
||||
|
||||
def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers
|
||||
return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY}
|
||||
|
||||
|
|
@ -94,10 +99,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
project_id=vertex_ai_project,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
return self._finalize_vertex_messages_environment(
|
||||
vertex_headers, vertex_api_base = self._finalize_vertex_messages_environment(
|
||||
headers,
|
||||
model,
|
||||
messages,
|
||||
optional_params,
|
||||
api_base,
|
||||
access_token,
|
||||
|
|
@ -105,6 +109,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
vertex_ai_project,
|
||||
vertex_ai_location,
|
||||
)
|
||||
return dict(self.headers_with_anthropic_betas(vertex_headers, optional_params, messages)), vertex_api_base
|
||||
|
||||
async def avalidate_anthropic_messages_environment(
|
||||
self,
|
||||
|
|
@ -122,10 +127,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
project_id=vertex_ai_project,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
return self._finalize_vertex_messages_environment(
|
||||
vertex_headers, vertex_api_base = self._finalize_vertex_messages_environment(
|
||||
headers,
|
||||
model,
|
||||
messages,
|
||||
optional_params,
|
||||
api_base,
|
||||
access_token,
|
||||
|
|
@ -133,12 +137,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
vertex_ai_project,
|
||||
vertex_ai_location,
|
||||
)
|
||||
return dict(self.headers_with_anthropic_betas(vertex_headers, optional_params, messages)), vertex_api_base
|
||||
|
||||
def _finalize_vertex_messages_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
messages: Sequence[object],
|
||||
optional_params: dict, # mutable-ok: preserves the existing provider optional-parameter contract
|
||||
api_base: str | None,
|
||||
access_token: str,
|
||||
|
|
@ -162,39 +166,48 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
|
||||
request_headers["content-type"] = "application/json"
|
||||
|
||||
# Add beta headers for Vertex AI
|
||||
tools: Final = optional_params.get("tools", [])
|
||||
beta_values: Final[set[str]] = set()
|
||||
|
||||
# Get existing beta headers if any
|
||||
existing_beta: Final = request_headers.get("anthropic-beta")
|
||||
if existing_beta:
|
||||
beta_values.update(b.strip() for b in existing_beta.split(","))
|
||||
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
|
||||
|
||||
# Check for web search tool
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value)
|
||||
break
|
||||
|
||||
# Check for tool search tools - Vertex AI uses different beta header
|
||||
anthropic_model_info: Final = AnthropicModelInfo()
|
||||
if anthropic_model_info.is_tool_search_used(tools):
|
||||
beta_values.add(get_tool_search_beta_header("vertex_ai"))
|
||||
|
||||
if optional_params.get("safeguards") is not None:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
|
||||
|
||||
if messages_carry_output_config(messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
|
||||
|
||||
if beta_values:
|
||||
request_headers["anthropic-beta"] = ",".join(beta_values)
|
||||
|
||||
return request_headers, api_base
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
tools: Final = tuple(
|
||||
tool for tool in map(as_str_mapping, iter_items(optional_params.get("tools") or ())) if tool is not None
|
||||
)
|
||||
existing_betas: Final = (headers.get("anthropic-beta") or "").split(",")
|
||||
beta_values: Final = frozenset(
|
||||
(
|
||||
*(beta.strip() for beta in existing_betas if beta.strip()),
|
||||
*context_management_betas(optional_params.get("context_management")), # pyright: ignore[reportArgumentType] # optional_params is the config contract's untyped request body
|
||||
*(
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value,)
|
||||
if any(map(_is_web_search, tools))
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(get_tool_search_beta_header("vertex_ai"),)
|
||||
if AnthropicModelInfo().is_tool_search_used(list(tools))
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,)
|
||||
if optional_params.get("safeguards") is not None
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value,)
|
||||
if messages_carry_output_config(messages)
|
||||
else ()
|
||||
),
|
||||
)
|
||||
)
|
||||
if not beta_values:
|
||||
return headers
|
||||
return {**headers, "anthropic-beta": ",".join(sorted(beta_values))}
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -336,7 +336,7 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
|
|||
|
||||
|
||||
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
|
||||
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format")
|
||||
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format", "context_management")
|
||||
|
||||
|
||||
class _CountTokensBodyOptions(TypedDict):
|
||||
|
|
@ -429,6 +429,7 @@ async def count_tokens(
|
|||
messages_format="anthropic",
|
||||
litellm_call_id=litellm_call_id,
|
||||
client_oauth_token=forwarded_anthropic_oauth_token(request.headers),
|
||||
anthropic_beta=request.headers.get("anthropic-beta"),
|
||||
**_count_tokens_body_options(data, auto_drop_params),
|
||||
)
|
||||
except HTTPException as e:
|
||||
|
|
|
|||
|
|
@ -72,6 +72,10 @@ class ProviderCountFailure:
|
|||
def denied_in_region(self) -> bool:
|
||||
return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES
|
||||
|
||||
@property
|
||||
def rejected_request(self) -> bool:
|
||||
return 400 <= self.status_code < 500 and self.status_code != 429
|
||||
|
||||
def with_hint(self, hint: str) -> "ProviderCountFailure":
|
||||
if self.rejected_before_provider_call:
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -14813,6 +14813,7 @@ async def _try_provider_token_count(
|
|||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
|
||||
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
|
||||
|
|
@ -14841,6 +14842,7 @@ async def _try_provider_token_count(
|
|||
context_management=context_management,
|
||||
chat_params=chat_params,
|
||||
request_options=request_options,
|
||||
anthropic_beta=anthropic_beta,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
return ProviderCountFailure(
|
||||
|
|
@ -14923,6 +14925,32 @@ async def _provider_token_count(
|
|||
return failure
|
||||
|
||||
|
||||
def _retried_without_rejected_options(
|
||||
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
|
||||
request_options: Mapping[str, JsonValue],
|
||||
) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]:
|
||||
if not request_options:
|
||||
return count_with
|
||||
return partial(_count_retrying_without_options, count_with)
|
||||
|
||||
|
||||
async def _count_retrying_without_options(
|
||||
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
|
||||
model: str,
|
||||
region: str | None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
counted: Final = await count_with(model, region)
|
||||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
|
||||
return counted
|
||||
verbose_proxy_logger.warning(
|
||||
"Token count for %s answered %s with the request's thinking, tool, and output options, "
|
||||
"so it counts again without them",
|
||||
model,
|
||||
counted.status_code,
|
||||
)
|
||||
return await count_with(model, region, request_options=MappingProxyType({}))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LocalTokenEstimate:
|
||||
total_tokens: int
|
||||
|
|
@ -15335,6 +15363,7 @@ async def count_request_tokens(
|
|||
litellm_call_id: str | None = None,
|
||||
client_oauth_token: str | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse:
|
||||
call_id: Final = resolve_litellm_call_id(litellm_call_id)
|
||||
if request.prompt is None and request.messages is None and request.contents is None:
|
||||
|
|
@ -15355,6 +15384,7 @@ async def count_request_tokens(
|
|||
responses_request=responses_request,
|
||||
client_oauth_token=client_oauth_token,
|
||||
request_options=request_options,
|
||||
anthropic_beta=anthropic_beta,
|
||||
)
|
||||
return await logged_token_count(
|
||||
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
|
||||
|
|
@ -15398,6 +15428,7 @@ async def _count_request_tokens(
|
|||
responses_request: Mapping[str, object] | None,
|
||||
client_oauth_token: str | None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -15474,6 +15505,7 @@ async def _count_request_tokens(
|
|||
)
|
||||
if isinstance(chat_count, ChatCountBridgeError):
|
||||
raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400)
|
||||
counted_options: Final = _count_request_options(request_options, count_deployment, sent_params)
|
||||
ladder: Final = (
|
||||
None
|
||||
if provider_counter is None
|
||||
|
|
@ -15485,27 +15517,33 @@ async def _count_request_tokens(
|
|||
routed_deployment,
|
||||
request.model,
|
||||
)
|
||||
or partial(
|
||||
_try_provider_token_count,
|
||||
provider_counter,
|
||||
custom_llm_provider,
|
||||
messages=(
|
||||
[dict(message) for message in chat_count.messages]
|
||||
if chat_count is not None
|
||||
else [{"role": "user", "content": prompt}]
|
||||
if prompt_as_message
|
||||
else request_messages
|
||||
or _retried_without_rejected_options(
|
||||
partial(
|
||||
_try_provider_token_count,
|
||||
provider_counter,
|
||||
custom_llm_provider,
|
||||
messages=(
|
||||
[dict(message) for message in chat_count.messages]
|
||||
if chat_count is not None
|
||||
else [{"role": "user", "content": prompt}]
|
||||
if prompt_as_message
|
||||
else request_messages
|
||||
),
|
||||
contents=None if prompt_as_message else contents,
|
||||
deployment=count_deployment,
|
||||
request_model=request.model,
|
||||
tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=_count_context_management(
|
||||
context_management, request.model, request_litellm_params
|
||||
),
|
||||
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
|
||||
request_options=counted_options,
|
||||
anthropic_beta=anthropic_beta,
|
||||
),
|
||||
contents=None if prompt_as_message else contents,
|
||||
deployment=count_deployment,
|
||||
request_model=request.model,
|
||||
tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=_count_context_management(context_management, request.model, request_litellm_params),
|
||||
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
|
||||
request_options=_count_request_options(request_options, count_deployment, sent_params),
|
||||
counted_options,
|
||||
),
|
||||
model_to_use=model_to_use,
|
||||
request_model=request.model,
|
||||
|
|
|
|||
|
|
@ -27,8 +27,12 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
|
|||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_ADVISOR_TOOL_TYPE,
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -488,7 +492,7 @@ async def _counted_requests(
|
|||
messages_format: TokenCountMessagesFormat,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
context_management: CountContextManagement | None = None,
|
||||
anthropic_beta: str | None = None,
|
||||
model: str = "claude-opus-5-5",
|
||||
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
|
@ -509,7 +513,7 @@ async def _counted_requests(
|
|||
messages_format=messages_format,
|
||||
chat_params=chat_params,
|
||||
request_options=request_options,
|
||||
context_management=context_management,
|
||||
anthropic_beta=anthropic_beta,
|
||||
)
|
||||
return result, tuple(sent.get() for _ in range(sent.qsize()))
|
||||
|
||||
|
|
@ -880,12 +884,6 @@ _MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
|
|||
_INFERENCE_PROVIDERS: Final = MappingProxyType(
|
||||
{AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"}
|
||||
)
|
||||
_CONTEXT_BETAS: Final = frozenset(
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
)
|
||||
)
|
||||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}
|
||||
_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000}
|
||||
|
||||
|
|
@ -894,6 +892,8 @@ async def _messages_inference_request(
|
|||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
inference_endpoint: str,
|
||||
request_options: Mapping[str, JsonValue],
|
||||
model: str = "claude-haiku-5-5",
|
||||
extra_headers: Mapping[str, str] = MappingProxyType({}),
|
||||
) -> httpx.Request:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
|
|
@ -904,12 +904,13 @@ async def _messages_inference_request(
|
|||
with respx.mock:
|
||||
respx.post(inference_endpoint).mock(side_effect=reply)
|
||||
await litellm.anthropic_messages(
|
||||
model=f"{_INFERENCE_PROVIDERS[counter_type]}/claude-haiku-5-5",
|
||||
model=f"{_INFERENCE_PROVIDERS[counter_type]}/{model}",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
max_tokens=4096,
|
||||
api_key="sk-ant-api03-test-key",
|
||||
api_base="https://gateway.example",
|
||||
extra_headers=dict(extra_headers),
|
||||
**request_options,
|
||||
)
|
||||
return sent.get()
|
||||
|
|
@ -923,16 +924,24 @@ def _betas(request: httpx.Request) -> frozenset[str]:
|
|||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "context_management"),
|
||||
("request_options", "expected_betas"),
|
||||
(
|
||||
({"thinking": _ENABLED_THINKING}, None),
|
||||
({"tool_choice": {"type": "any"}}, None),
|
||||
({"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, None),
|
||||
({"thinking": _ENABLED_THINKING}, frozenset()),
|
||||
({"tool_choice": {"type": "any"}}, frozenset()),
|
||||
(
|
||||
{"thinking": _ENABLED_THINKING},
|
||||
{"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]},
|
||||
{"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}},
|
||||
frozenset((ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value,)),
|
||||
),
|
||||
({}, {"edits": [{"type": "compact_20260112"}]}),
|
||||
(
|
||||
{
|
||||
"thinking": _ENABLED_THINKING,
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
},
|
||||
},
|
||||
frozenset((ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,)),
|
||||
),
|
||||
({"context_management": {"edits": [{"type": "compact_20260112"}]}}, frozenset()),
|
||||
),
|
||||
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
|
||||
)
|
||||
|
|
@ -940,12 +949,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
|
|||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
request_options: dict[str, JsonValue],
|
||||
context_management: dict[str, JsonValue] | None,
|
||||
expected_betas: frozenset[str],
|
||||
) -> None:
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type,
|
||||
count_endpoint.removesuffix("/count_tokens"),
|
||||
{**request_options, **({} if context_management is None else {"context_management": context_management})},
|
||||
counter_type, count_endpoint.removesuffix("/count_tokens"), request_options
|
||||
)
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
|
|
@ -954,9 +961,6 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
|
|||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
request_options=request_options,
|
||||
context_management=(
|
||||
None if context_management is None else CountContextManagement(spec=context_management, drop_invalid_edits=False)
|
||||
),
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
|
|
@ -973,7 +977,38 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
|
|||
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
|
||||
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
|
||||
}
|
||||
assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS
|
||||
assert expected_betas <= _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_sends_the_thinking_display_and_caller_betas_messages_inference_sends(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
caller_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
|
||||
display_thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type,
|
||||
count_endpoint.removesuffix("/count_tokens"),
|
||||
display_thinking,
|
||||
model="claude-opus-4-6",
|
||||
extra_headers={"anthropic-beta": caller_beta},
|
||||
)
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
request_options=display_thinking,
|
||||
anthropic_beta=caller_beta,
|
||||
model="claude-opus-4-6",
|
||||
)
|
||||
|
||||
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
|
||||
assert counted_betas == _betas(inference)
|
||||
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import base64
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.bedrock.count_tokens.transformation import (
|
||||
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS,
|
||||
|
|
@ -53,6 +56,28 @@ def test_detect_input_type_converse_blocks_route_to_converse():
|
|||
assert config._detect_input_type(request) == "converse"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"option",
|
||||
(
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
{"tool_choice": {"type": "auto"}},
|
||||
{"output_config": {"effort": "low"}},
|
||||
),
|
||||
)
|
||||
def test_pass_through_body_with_string_content_is_counted_as_converse_whatever_options_it_carries(
|
||||
option: dict[str, object],
|
||||
) -> None:
|
||||
request: Final = {
|
||||
"model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
**option,
|
||||
}
|
||||
|
||||
result: Final = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request)
|
||||
|
||||
assert result["input"] == {"converse": {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}}
|
||||
|
||||
|
||||
def test_transform_to_invoke_model_format_base64_encodes_body():
|
||||
"""The CountTokens API expects invokeModel.body as a base64-encoded blob;
|
||||
Anthropic Messages bodies additionally need anthropic_version/max_tokens
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ from pydantic import JsonValue, TypeAdapter
|
|||
import litellm
|
||||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.utils import CountContextManagement
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_EFFORT_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
|
||||
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
|
|
@ -350,29 +353,36 @@ _THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
|
|||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
|
||||
|
||||
|
||||
async def _messages_inference_and_count(deployment_model: str, model_to_use: str) -> tuple[_Bedrock, _Bedrock]:
|
||||
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
|
||||
|
||||
|
||||
async def _messages_inference_and_count(
|
||||
deployment_model: str,
|
||||
model_to_use: str,
|
||||
request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS,
|
||||
tool: Mapping[str, JsonValue] = _WEATHER_TOOL,
|
||||
runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}),
|
||||
) -> tuple[_Bedrock, _Bedrock]:
|
||||
credentials: Final = DEPLOYMENT["litellm_params"]
|
||||
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=MESSAGES,
|
||||
tools=[_WEATHER_TOOL],
|
||||
tools=[dict(tool)],
|
||||
max_tokens=4096,
|
||||
context_management=_CONTEXT_EDITS,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
|
||||
**_THINKING_AND_TOOL_CHOICE,
|
||||
**request_options,
|
||||
**credentials,
|
||||
)
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177}))
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use=model_to_use,
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {**credentials, "model": deployment_model}},
|
||||
request_model="claude",
|
||||
tools=[_WEATHER_TOOL],
|
||||
request_options=_THINKING_AND_TOOL_CHOICE,
|
||||
context_management=CountContextManagement(spec=_CONTEXT_EDITS, drop_invalid_edits=False),
|
||||
tools=[dict(tool)],
|
||||
request_options=request_options,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.error is False
|
||||
|
|
@ -419,6 +429,79 @@ async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_infe
|
|||
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
|
||||
|
||||
|
||||
_DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
|
||||
|
||||
|
||||
def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
|
||||
return tuple(sorted(request.headers["anthropic-beta"].split(",")))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "request_options", "expected_beta"),
|
||||
(
|
||||
(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
|
||||
{"output_config": {"effort": "low"}},
|
||||
ANTHROPIC_EFFORT_BETA_HEADER,
|
||||
),
|
||||
("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER),
|
||||
),
|
||||
ids=("effort", "thinking_display"),
|
||||
)
|
||||
async def test_runtime_count_sends_the_betas_invoke_inference_sends(
|
||||
deployment_model: str, request_options: dict[str, JsonValue], expected_beta: str
|
||||
) -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta")
|
||||
assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
|
||||
assert isinstance(counted_betas, list) and expected_beta in counted_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
||||
"us.anthropic.claude-opus-5-5",
|
||||
_DISPLAY_THINKING,
|
||||
runtime=(400, UNSUPPORTED),
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
|
||||
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
|
||||
assert isinstance(sent_betas, list)
|
||||
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas))
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in sent_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (MANTLE_HOST,)
|
||||
assert _sent_betas(bedrock.requests[0]) == _sent_betas(inference.requests[0])
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
||||
"us.anthropic.claude-opus-5-5",
|
||||
_THINKING_AND_TOOL_CHOICE,
|
||||
tool={**_WEATHER_TOOL, "eager_input_streaming": True},
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
|
||||
|
||||
|
||||
_CITY_SCHEMA: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
|
|
|
|||
|
|
@ -2016,8 +2016,7 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
|
|||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=tools,
|
||||
request_options=options,
|
||||
context_management=CountContextManagement(spec=context_edits, drop_invalid_edits=False),
|
||||
request_options={**options, "context_management": context_edits},
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
|||
kimi_k3_tokens,
|
||||
load_kimi_k3_encoding,
|
||||
)
|
||||
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensInputType
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
|
||||
from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http
|
||||
|
|
@ -368,6 +369,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "claude-3-sonnet-20240229"
|
||||
|
|
@ -451,6 +453,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "gpt-4"
|
||||
|
|
@ -940,6 +943,7 @@ class _FailingRuntimeHandler(BedrockCountTokensHandler):
|
|||
litellm_params: dict[str, object],
|
||||
resolved_model: str,
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
input_type: BedrockCountTokensInputType | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
raise self._error
|
||||
|
||||
|
|
@ -1272,6 +1276,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"detail":{"message":"Input is too long for requested model."}}',
|
||||
|
|
@ -1332,6 +1337,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"Message":"Bearer Token has expired"}',
|
||||
|
|
@ -1392,6 +1398,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
|
|
@ -1416,7 +1423,9 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
proxy_server.count_request_tokens = original_token_counter
|
||||
|
||||
|
||||
def _json_request(body: Mapping[str, object], user_agent: str) -> Request:
|
||||
def _json_request(
|
||||
body: Mapping[str, object], user_agent: str, headers: Mapping[str, str] = MappingProxyType({})
|
||||
) -> Request:
|
||||
payload: Final = json.dumps(body).encode()
|
||||
|
||||
async def receive() -> Message:
|
||||
|
|
@ -1428,7 +1437,11 @@ def _json_request(body: Mapping[str, object], user_agent: str) -> Request:
|
|||
"method": "POST",
|
||||
"path": "/v1/messages/count_tokens",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json"), (b"user-agent", user_agent.encode())],
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"user-agent", user_agent.encode()),
|
||||
*((name.encode(), value.encode()) for name, value in headers.items()),
|
||||
],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
|
|
@ -1641,10 +1654,11 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("additional_drop_params", "expected_paths"),
|
||||
("additional_drop_params", "request_params", "expected_paths"),
|
||||
(
|
||||
(
|
||||
None,
|
||||
{},
|
||||
{
|
||||
"thinking",
|
||||
"tool_choice",
|
||||
|
|
@ -1656,17 +1670,32 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
|
|||
),
|
||||
(
|
||||
["tool_choice", "output_config.format"],
|
||||
{},
|
||||
{"thinking", "output_config", "output_config.effort", "context_management"},
|
||||
),
|
||||
(
|
||||
["context_management"],
|
||||
{},
|
||||
{"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"},
|
||||
),
|
||||
(
|
||||
None,
|
||||
{"additional_drop_params": ["thinking", "context_management"]},
|
||||
{"tool_choice", "output_config", "output_config.effort", "output_config.format"},
|
||||
),
|
||||
),
|
||||
ids=(
|
||||
"nothing-dropped",
|
||||
"deployment-drops-tool-choice-and-the-format",
|
||||
"deployment-drops-context-management",
|
||||
"request-drops-thinking-and-context-management",
|
||||
),
|
||||
ids=("nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management"),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
|
||||
monkeypatch: pytest.MonkeyPatch, additional_drop_params: list[str] | None, expected_paths: set[str]
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
additional_drop_params: list[str] | None,
|
||||
request_params: dict[str, JsonValue],
|
||||
expected_paths: set[str],
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
|
|
@ -1688,6 +1717,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
|
|||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
|
||||
**_CLAUDE_COUNT_OPTIONS,
|
||||
**request_params,
|
||||
},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
|
|
@ -1709,6 +1739,187 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
|
|||
)
|
||||
|
||||
|
||||
_CALLER_BETA: Final = "web-fetch-2025-09-10"
|
||||
|
||||
|
||||
async def _count_on_a_claude_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_model: str,
|
||||
body: Mapping[str, JsonValue],
|
||||
reply: Callable[[httpx.Request], httpx.Response],
|
||||
user_agent: str = "python-httpx/0.28.1",
|
||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "claude-agent", "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test"}}
|
||||
]
|
||||
)
|
||||
request: Final = _json_request(
|
||||
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
||||
user_agent,
|
||||
{"anthropic-beta": _CALLER_BETA},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(side_effect=reply)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
return response, tuple(call.request for call in route.calls)
|
||||
|
||||
|
||||
def _counts_only_without_options(request: httpx.Request) -> httpx.Response:
|
||||
if _sent_option_paths(json.loads(request.content)):
|
||||
return httpx.Response(
|
||||
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}}
|
||||
)
|
||||
return httpx.Response(200, json={"input_tokens": 7})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "options", "rejected_by_anthropic"),
|
||||
(
|
||||
("anthropic/claude-haiku-4-5", {"output_config": {"effort": "fastest"}}, False),
|
||||
("anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, True),
|
||||
),
|
||||
ids=("rejected-by-the-count-transform", "rejected-by-anthropic"),
|
||||
)
|
||||
async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_claude_count_rejects_them(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_model: str,
|
||||
options: dict[str, JsonValue],
|
||||
rejected_by_anthropic: bool,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch, deployment_model, options, _counts_only_without_options
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert response["input_tokens"] == 7
|
||||
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
|
||||
*((True,) if rejected_by_anthropic else ()),
|
||||
False,
|
||||
]
|
||||
assert [_CALLER_BETA in sent.headers.get("anthropic-beta", "") for sent in requests] == [
|
||||
*((True,) if rejected_by_anthropic else ()),
|
||||
False,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status_code", (401, 403, 404))
|
||||
async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_options_also_gets(
|
||||
monkeypatch: pytest.MonkeyPatch, status_code: int
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-haiku-5-5",
|
||||
_CLAUDE_COUNT_OPTIONS,
|
||||
lambda _: httpx.Response(status_code, json={"type": "error", "error": {"type": "error", "message": "denied"}}),
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == status_code
|
||||
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [True, False]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-haiku-5-5",
|
||||
_CLAUDE_COUNT_OPTIONS,
|
||||
lambda _: httpx.Response(429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow"}}),
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 429
|
||||
assert len(requests) == 1
|
||||
|
||||
|
||||
async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
await litellm.anthropic_messages(
|
||||
model=f"anthropic/{model}",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
max_tokens=4096,
|
||||
api_key="sk-ant-test",
|
||||
**options,
|
||||
)
|
||||
return json.loads(route.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_drop_params", "user_agent"),
|
||||
(({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send(
|
||||
monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
|
||||
dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True})
|
||||
kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking})
|
||||
assert dropped["thinking"] != kept["thinking"]
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-opus-4-7",
|
||||
{"thinking": thinking, **request_drop_params},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
user_agent,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
counted: Final = json.loads(count.content)
|
||||
assert {name: counted.get(name) for name in ("thinking", "output_config")} == {
|
||||
name: dropped.get(name) for name in ("thinking", "output_config")
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
output_format: Final[dict[str, JsonValue]] = {
|
||||
"type": "json_schema",
|
||||
"schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
}
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-haiku-5-5",
|
||||
{"output_format": output_format},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
counted: Final = json.loads(count.content)
|
||||
assert "output_format" not in counted
|
||||
assert counted["output_config"]["format"] == output_format
|
||||
|
||||
|
||||
class _ScriptedCounter(BaseTokenCounter):
|
||||
def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None:
|
||||
self._replies = replies
|
||||
|
|
@ -1734,6 +1945,7 @@ class _ScriptedCounter(BaseTokenCounter):
|
|||
context_management: object = None,
|
||||
chat_params: object = None,
|
||||
request_options: object = None,
|
||||
anthropic_beta: object = None,
|
||||
) -> TokenCountResponse | None:
|
||||
self.calls.append((model_to_use, messages, tools, system))
|
||||
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue