mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): forward context_management, thinking, tool_choice and output_config to the Claude count APIs
Backport of the count_tokens context_management fix (f012c60033,875a0e6404,a6f4acdb6f,408372dbcb,8db6f0d5ed,f84c244f3e,14b572d44b,502ce4a887and0fa2f90ab6on litellm_count_tokens_forward_context_management) to lts/1.108.x, adapted to the line's count path. The count sends inference's betas and tools when it forwards options, counts with the deployment's model, adds the effort beta on Bedrock runtime counts, strips disable_parallel_tool_use on Converse the way Converse inference does, lets the caller's drop_params replace the deployment's like inference (an explicit null included), and counts once more without the forwarded options when the provider rejects them with a 4xx other than 401, 403 or 429
This commit is contained in:
parent
e03de97a98
commit
30bfb30dd0
30 changed files with 2452 additions and 186 deletions
|
|
@ -12,7 +12,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
|||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -43,6 +43,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -345,6 +346,18 @@ def _beta_header_values(side: str | Sequence[str] | None) -> tuple[str, ...]:
|
|||
return tuple(entry for entry in side if isinstance(entry, str))
|
||||
|
||||
|
||||
def context_management_betas(context_management: JsonValue) -> frozenset[str]:
|
||||
edits: Final = context_management.get("edits") if isinstance(context_management, dict) else None
|
||||
if not isinstance(edits, (list, tuple)):
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
|
||||
if isinstance(edit, dict) and edit.get("type") == "compact_20260112"
|
||||
else ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
|
||||
for edit in edits
|
||||
)
|
||||
|
||||
|
||||
def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta: str | Sequence[str] | None) -> str:
|
||||
"""Merge anthropic-beta header values, deduplicated and sorted.
|
||||
|
||||
|
|
@ -1283,8 +1296,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Anthropic CountTokens API handler.
|
|||
Uses httpx for HTTP requests instead of the Anthropic SDK.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
betas: Sequence[str] = (),
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
Handle a CountTokens request using httpx.
|
||||
|
|
@ -79,7 +80,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
verbose_logger.debug("Making request to: %s", endpoint_url)
|
||||
|
||||
# Get required headers
|
||||
headers: Final = self.get_count_tokens_headers(auth_header)
|
||||
headers: Final = self.get_count_tokens_headers(auth_header, betas)
|
||||
|
||||
# Use LiteLLM's async httpx client
|
||||
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
Anthropic Token Counter implementation using the CountTokens API.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
|
@ -10,6 +12,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, claude_count_options
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
|
@ -38,6 +41,8 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "anthropic",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
Count tokens using Anthropic's CountTokens API.
|
||||
|
|
@ -57,6 +62,12 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
if not messages:
|
||||
return None
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.ANTHROPIC, messages, tools, request_options, anthropic_beta, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "anthropic_api")
|
||||
|
||||
deployment = deployment or {}
|
||||
litellm_params: Final = deployment.get("litellm_params", {})
|
||||
api_base: Final = litellm_params.get("api_base")
|
||||
|
|
@ -84,11 +95,17 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
|
||||
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
|
||||
model=model_to_use,
|
||||
messages=messages,
|
||||
messages=messages
|
||||
if claude_options.messages is None
|
||||
else [dict(message) for message in claude_options.messages],
|
||||
auth_header=auth_header,
|
||||
api_base=api_base,
|
||||
tools=tools,
|
||||
tools=tools
|
||||
if claude_options.messages is None
|
||||
else [dict(tool) for tool in claude_options.tools or ()] or None,
|
||||
system=system,
|
||||
optional_params=claude_options.options,
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -4,11 +4,14 @@ Anthropic CountTokens API transformation logic.
|
|||
This module handles the transformation of requests to Anthropic's CountTokens API format.
|
||||
"""
|
||||
|
||||
import copy
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
|
|
@ -17,12 +20,17 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
)
|
||||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.types.llms.openai import ChatCompletionImageObject
|
||||
from litellm.types.utils import LlmProviders, TokenCountResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
|
||||
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
|
||||
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
|
||||
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
|
||||
|
||||
def _count_image(block: JsonValue) -> JsonValue:
|
||||
|
|
@ -74,6 +82,202 @@ def _lift_leading_system(
|
|||
return conversation, lifted or system
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChatCountTranslationError:
|
||||
message: str
|
||||
|
||||
def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse:
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
request_model=request_model,
|
||||
model_used=model_used,
|
||||
tokenizer_type=tokenizer_type,
|
||||
error=True,
|
||||
error_message=self.message,
|
||||
status_code=400,
|
||||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore"))
|
||||
class _CountedOptions(TypedDict):
|
||||
thinking: NotRequired[ReadOnly[JsonValue]]
|
||||
tool_choice: NotRequired[ReadOnly[JsonValue]]
|
||||
output_config: NotRequired[ReadOnly[JsonValue]]
|
||||
output_format: NotRequired[ReadOnly[JsonValue]]
|
||||
context_management: NotRequired[ReadOnly[JsonValue]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore"))
|
||||
class _AnthropicChatRequest(_CountedOptions):
|
||||
messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]]
|
||||
tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]]
|
||||
|
||||
|
||||
_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest)
|
||||
_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_HEADERS: Final = TypeAdapter(dict[str, str])
|
||||
|
||||
|
||||
def _count_options(request: _CountedOptions) -> Mapping[str, JsonValue]:
|
||||
output_config: Final = request.get("output_config")
|
||||
output_format: Final = request.get("output_format")
|
||||
counted_config: Final = {
|
||||
**(output_config if isinstance(output_config, dict) else {}),
|
||||
**({} if output_format is None else {"format": output_format}),
|
||||
}
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in (
|
||||
("thinking", request.get("thinking")),
|
||||
("tool_choice", request.get("tool_choice")),
|
||||
("output_config", counted_config or None),
|
||||
("context_management", request.get("context_management")),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ClaudeCountOptions:
|
||||
options: Mapping[str, JsonValue]
|
||||
betas: tuple[str, ...]
|
||||
messages: tuple[Mapping[str, JsonValue], ...] | None
|
||||
tools: tuple[Mapping[str, JsonValue], ...] | None
|
||||
|
||||
|
||||
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
|
||||
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
|
||||
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
|
||||
|
||||
|
||||
_STATUSES_NOT_CAUSED_BY_THE_REQUEST: Final = frozenset({401, 403, 429})
|
||||
|
||||
|
||||
def is_request_rejection(status_code: int) -> bool:
|
||||
return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST
|
||||
|
||||
|
||||
def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None:
|
||||
deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {})
|
||||
deployment_model: Final = deployment_params.get("model")
|
||||
if not isinstance(deployment_model, str):
|
||||
return None
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
provider_hint: Final = deployment_params.get("custom_llm_provider")
|
||||
api_base: Final = deployment_params.get("api_base")
|
||||
inference_model, _, _, _ = get_llm_provider(
|
||||
model=deployment_model,
|
||||
custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
)
|
||||
return inference_model
|
||||
|
||||
|
||||
def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None:
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
|
||||
return None if messages_config is None else (model, messages_config)
|
||||
|
||||
|
||||
def _transform_litellm_params(deployment: Mapping[str, object] | None) -> "GenericLiteLLMParams":
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
resolved_deployment: Final = deployment or {}
|
||||
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
return GenericLiteLLMParams.model_validate(
|
||||
{
|
||||
"drop_params": deployment_params.get("drop_params"),
|
||||
"model_info": resolved_deployment.get("model_info"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _claude_count_options_for(
|
||||
messages_config: BaseAnthropicMessagesConfig,
|
||||
model: str,
|
||||
litellm_params: "GenericLiteLLMParams",
|
||||
provider: LlmProviders,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
sent: Mapping[str, JsonValue],
|
||||
anthropic_beta: str | None,
|
||||
) -> ClaudeCountOptions:
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
|
||||
leading_system, conversation = split_leading_system_run(messages)
|
||||
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
||||
sent_params: Final = {
|
||||
**copy.deepcopy(sent),
|
||||
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
|
||||
}
|
||||
beta_headers: Final = messages_config.headers_with_anthropic_betas(
|
||||
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
|
||||
)
|
||||
sent_headers: Final = (
|
||||
_HEADERS.validate_python(
|
||||
update_headers_with_filtered_beta(
|
||||
headers=dict(beta_headers),
|
||||
provider=provider.value,
|
||||
keep=messages_config.model_specific_anthropic_betas(model, litellm_params),
|
||||
)
|
||||
)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else dict(beta_headers)
|
||||
)
|
||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||
model=model,
|
||||
messages=sent_conversation,
|
||||
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
|
||||
litellm_params=litellm_params,
|
||||
headers=sent_headers,
|
||||
)
|
||||
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
|
||||
return ClaudeCountOptions(
|
||||
options=_count_options(transformed),
|
||||
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
|
||||
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
|
||||
tools=transformed.get("tools"),
|
||||
)
|
||||
|
||||
|
||||
def claude_count_options(
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
request_options: Mapping[str, JsonValue],
|
||||
anthropic_beta: str | None,
|
||||
deployment: Mapping[str, object] | None,
|
||||
) -> ClaudeCountOptions | ChatCountTranslationError:
|
||||
sent: Final = {name: value for name, value in request_options.items() if value is not None}
|
||||
if not sent:
|
||||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
inference_model: Final = deployment_inference_model(deployment)
|
||||
transform: Final = (
|
||||
None if inference_model is None else _messages_transform(inference_model, provider)
|
||||
) or _messages_transform(model, provider)
|
||||
if transform is None:
|
||||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
transform_model, messages_config = transform
|
||||
litellm_params: Final = _transform_litellm_params(deployment)
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
|
||||
try:
|
||||
return _claude_count_options_for(
|
||||
messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
except (AttributeError, KeyError, TypeError, ValueError) as e:
|
||||
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
|
||||
|
||||
|
||||
class AnthropicCountTokensConfig:
|
||||
"""
|
||||
Configuration and transformation logic for Anthropic CountTokens API.
|
||||
|
|
@ -137,7 +341,7 @@ class AnthropicCountTokensConfig:
|
|||
)
|
||||
)
|
||||
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str], betas: Sequence[str] = ()) -> dict[str, str]:
|
||||
"""The count-tokens headers around a resolved Anthropic auth header
|
||||
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
|
||||
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
|
||||
|
|
@ -147,7 +351,7 @@ class AnthropicCountTokensConfig:
|
|||
"anthropic-version": "2023-06-01",
|
||||
**auth_header,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
auth_header.get("anthropic-beta"), (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *betas)
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from ...common_utils import (
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
merge_anthropic_beta_headers,
|
||||
optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta,
|
||||
|
|
@ -378,6 +379,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
messages=messages,
|
||||
)
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return self._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared beta step takes and returns bare dicts
|
||||
headers=dict(headers),
|
||||
optional_params=dict(optional_params),
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _translate_reasoning_effort_to_anthropic(
|
||||
model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str
|
||||
|
|
@ -742,28 +755,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", ())
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
|
||||
|
||||
# Check for structured outputs. Anthropic's newer request shape nests
|
||||
# the schema under output_config.format; the older top-level
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Azure AI Anthropic CountTokens API handler.
|
|||
Uses httpx for HTTP requests with Azure authentication.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -12,7 +13,7 @@ from pydantic import JsonValue
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, merge_anthropic_beta_headers
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
|
|
@ -36,6 +37,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: object = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
betas: Sequence[str] = (),
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Handle a CountTokens request using httpx with Azure authentication.
|
||||
|
|
@ -66,6 +69,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
verbose_logger.debug("Transformed request: %s", request_body)
|
||||
|
|
@ -98,7 +102,10 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
|
||||
response: Final = await async_client.post(
|
||||
endpoint_url,
|
||||
headers=headers,
|
||||
headers={
|
||||
**headers,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(headers.get("anthropic-beta"), betas),
|
||||
},
|
||||
json=request_body,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API.
|
|||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, claude_count_options
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
|
||||
AzureAIAnthropicCountTokensHandler,
|
||||
)
|
||||
|
|
@ -55,6 +58,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "anthropic",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
Count tokens using Azure AI Anthropic's CountTokens API.
|
||||
|
|
@ -74,6 +79,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
if not messages:
|
||||
return None
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.AZURE_AI, messages, tools, request_options, anthropic_beta, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
||||
|
||||
deployment = deployment or {}
|
||||
litellm_params: Final = deployment.get("litellm_params", {})
|
||||
|
||||
|
|
@ -95,12 +106,18 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
try:
|
||||
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
|
||||
model=model_to_use,
|
||||
messages=messages,
|
||||
messages=messages
|
||||
if claude_options.messages is None
|
||||
else [dict(message) for message in claude_options.messages],
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
tools=tools,
|
||||
tools=tools
|
||||
if claude_options.messages is None
|
||||
else [dict(tool) for tool in claude_options.tools or ()] or None,
|
||||
system=system,
|
||||
optional_params=claude_options.options,
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -127,6 +127,18 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
"""
|
||||
return headers, None
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return headers
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
beta_header: Final = next((value for name, value in headers.items() if name.lower() == "anthropic-beta"), "")
|
||||
return tuple(sorted(frozenset(beta.strip() for beta in beta_header.split(",")) - {""}))
|
||||
|
||||
def should_filter_anthropic_beta_headers(self) -> bool:
|
||||
"""
|
||||
Whether ``anthropic-beta`` header values should be filtered down to the
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import json
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals
|
||||
|
|
@ -39,6 +40,8 @@ class BaseTokenCounter(ABC):
|
|||
messages_format: TokenCountMessagesFormat = "openai",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -2,20 +2,30 @@
|
|||
Bedrock Token Counter implementation using the CountTokens API.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import JsonValue
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
NO_CLAUDE_COUNT_OPTIONS,
|
||||
ChatCountTranslationError,
|
||||
ClaudeCountOptions,
|
||||
claude_count_options,
|
||||
deployment_inference_model,
|
||||
is_request_rejection,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
|
||||
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
|
||||
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
|
||||
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER
|
||||
from litellm.types.utils import (
|
||||
CountContextManagement,
|
||||
LiteLLMPydanticObjectBase,
|
||||
|
|
@ -58,6 +68,48 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
|
|||
)
|
||||
|
||||
|
||||
def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options: bool) -> bool:
|
||||
return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code)
|
||||
|
||||
|
||||
def _invoke_fields(options: Mapping[str, JsonValue], betas: Sequence[str]) -> Mapping[str, object]:
|
||||
return {**options, **({"anthropic_beta": list(betas)} if betas else {})}
|
||||
|
||||
|
||||
def _sets_effort(options: Mapping[str, JsonValue]) -> bool:
|
||||
output_config: Final = options.get("output_config")
|
||||
return isinstance(output_config, dict) and "effort" in output_config
|
||||
|
||||
|
||||
def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]:
|
||||
if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas:
|
||||
return claude_options.betas
|
||||
return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER)
|
||||
|
||||
|
||||
def _runs_on_converse(inference_model: str) -> bool:
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
|
||||
return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(inference_model) is None
|
||||
|
||||
|
||||
def _options_converse_sends(
|
||||
request_options: Mapping[str, JsonValue], inference_model: str | None
|
||||
) -> Mapping[str, JsonValue]:
|
||||
tool_choice: Final = request_options.get("tool_choice")
|
||||
if (
|
||||
not isinstance(tool_choice, dict)
|
||||
or "disable_parallel_tool_use" not in tool_choice
|
||||
or inference_model is None
|
||||
or not _runs_on_converse(inference_model)
|
||||
):
|
||||
return request_options
|
||||
return {
|
||||
**request_options,
|
||||
"tool_choice": {key: value for key, value in tool_choice.items() if key != "disable_parallel_tool_use"},
|
||||
}
|
||||
|
||||
|
||||
class BedrockTokenCounter(BaseTokenCounter):
|
||||
"""Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model"""
|
||||
|
||||
|
|
@ -77,21 +129,19 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
) -> bool:
|
||||
return custom_llm_provider == LlmProviders.BEDROCK.value
|
||||
|
||||
async def _count_with(
|
||||
self,
|
||||
handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler,
|
||||
tokenizer_type: str,
|
||||
request_data: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
resolved_model: str,
|
||||
) -> CountTokensOutcome:
|
||||
def _count_on_mantle(
|
||||
self, request_data: Mapping[str, object], litellm_params: Mapping[str, object], resolved_model: str
|
||||
) -> Awaitable[Mapping[str, JsonValue]]:
|
||||
return self._mantle_handler.handle_count_tokens_request(
|
||||
request_data=dict(request_data),
|
||||
litellm_params=dict(litellm_params),
|
||||
resolved_model=resolved_model,
|
||||
client=self._client,
|
||||
)
|
||||
|
||||
async def _count_with(self, count: Awaitable[Mapping[str, JsonValue]], tokenizer_type: str) -> CountTokensOutcome:
|
||||
try:
|
||||
result: Final = await handler.handle_count_tokens_request(
|
||||
request_data=request_data,
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
client=self._client,
|
||||
)
|
||||
result: Final = await count
|
||||
reply: Final = _CountTokensReply.model_validate(result)
|
||||
except BedrockError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -115,32 +165,74 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "anthropic",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
use_mantle: Final = model_to_use.startswith("bedrock_mantle/")
|
||||
counted_model: Final = model_to_use.removeprefix("bedrock_mantle/")
|
||||
resolved_model: Final = get_bedrock_base_model(counted_model)
|
||||
primary_tokenizer: Final = MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE
|
||||
litellm_params: Final = (deployment or {}).get("litellm_params", {})
|
||||
request_data: Final[dict[str, object]] = {
|
||||
claude_options: Final = (
|
||||
claude_count_options(
|
||||
counted_model,
|
||||
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
|
||||
messages,
|
||||
tools,
|
||||
_options_converse_sends(request_options, deployment_inference_model(deployment)),
|
||||
anthropic_beta,
|
||||
deployment,
|
||||
)
|
||||
if is_mantle_claude_model(resolved_model)
|
||||
else NO_CLAUDE_COUNT_OPTIONS
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, primary_tokenizer)
|
||||
invoke_fields: Final = _invoke_fields(claude_options.options, claude_options.betas)
|
||||
sent_tools: Final = tools if claude_options.messages is None else claude_options.tools
|
||||
counted_request: Final[Mapping[str, object]] = {
|
||||
"model": model_to_use,
|
||||
"messages": messages,
|
||||
**({"tools": tools} if tools else {}),
|
||||
"messages": messages
|
||||
if claude_options.messages is None
|
||||
else [dict(message) for message in claude_options.messages],
|
||||
**({"tools": [dict(tool) for tool in sent_tools]} if sent_tools else {}),
|
||||
**({"system": system} if system else {}),
|
||||
}
|
||||
use_mantle: Final = model_to_use.startswith("bedrock_mantle/")
|
||||
resolved_model: Final = get_bedrock_base_model(model_to_use.removeprefix("bedrock_mantle/"))
|
||||
mantle_request: Final = {**counted_request, **invoke_fields}
|
||||
runtime_request: Final = {
|
||||
**counted_request,
|
||||
**_invoke_fields(claude_options.options, _runtime_count_betas(claude_options)),
|
||||
}
|
||||
|
||||
primary: Final = await self._count_with(
|
||||
self._mantle_handler if use_mantle else self._runtime_handler,
|
||||
MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE,
|
||||
request_data,
|
||||
litellm_params,
|
||||
resolved_model,
|
||||
)
|
||||
outcome: Final = (
|
||||
primary: Final = (
|
||||
await self._count_with(
|
||||
self._mantle_handler, MANTLE_TOKENIZER_TYPE, request_data, litellm_params, resolved_model
|
||||
self._count_on_mantle(mantle_request, litellm_params, resolved_model), primary_tokenizer
|
||||
)
|
||||
if use_mantle
|
||||
else await self._count_with(
|
||||
self._runtime_handler.handle_count_tokens_request(
|
||||
request_data=runtime_request,
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
client=self._client,
|
||||
input_type="invokeModel" if invoke_fields else None,
|
||||
),
|
||||
primary_tokenizer,
|
||||
)
|
||||
)
|
||||
fallback: Final = (
|
||||
await self._count_with(
|
||||
self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE
|
||||
)
|
||||
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
|
||||
else None
|
||||
)
|
||||
outcome: Final = (
|
||||
fallback
|
||||
if fallback is not None and _mantle_answered_for_runtime(fallback, forwarded_options=bool(invoke_fields))
|
||||
else primary
|
||||
)
|
||||
match outcome:
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.bedrock.base_aws_llm import run_aws_signing
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
|
||||
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig, BedrockCountTokensInputType
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
|
||||
|
||||
|
||||
|
|
@ -29,6 +29,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
litellm_params: dict[str, Any],
|
||||
resolved_model: str,
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
input_type: BedrockCountTokensInputType | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Handle a CountTokens request using existing LiteLLM patterns.
|
||||
|
|
@ -43,7 +44,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
"""
|
||||
try:
|
||||
# Validate the request
|
||||
self.validate_count_tokens_request(request_data)
|
||||
self.validate_count_tokens_request(request_data, input_type)
|
||||
|
||||
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
|
||||
|
||||
|
|
@ -57,7 +58,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
verbose_logger.debug("Retrieved AWS region: %s", aws_region_name)
|
||||
|
||||
# Transform request to Bedrock format (supports both Converse and InvokeModel)
|
||||
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data)
|
||||
bedrock_request: Final = self.transform_anthropic_to_bedrock_count_tokens(
|
||||
request_data=request_data, input_type=input_type
|
||||
)
|
||||
|
||||
verbose_logger.debug("Transformed request: %s", bedrock_request)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -6,7 +7,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountTokensConfig
|
||||
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
|
||||
|
|
@ -23,6 +24,8 @@ class MantleCountTokensRequest(TypedDict):
|
|||
|
||||
_COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
|
||||
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
|
||||
_BETAS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class BedrockMantleCountTokensHandler(BaseAWSLLM):
|
||||
|
|
@ -61,13 +64,21 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
|
|||
messages=request["messages"],
|
||||
tools=request.get("tools"),
|
||||
system=request.get("system"),
|
||||
optional_params=_COUNT_OPTIONS.validate_python(
|
||||
{name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}
|
||||
),
|
||||
)
|
||||
verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
|
||||
api_key: Final = litellm_params.get("api_key")
|
||||
betas: Final = _BETAS.validate_python(request_data.get("anthropic_beta", ()))
|
||||
signed_headers, signed_body = await run_aws_signing(
|
||||
self._sign_request,
|
||||
service_name="bedrock",
|
||||
headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": MANTLE_ANTHROPIC_VERSION,
|
||||
**({"anthropic-beta": ",".join(betas)} if betas else {}),
|
||||
},
|
||||
optional_params=litellm_params,
|
||||
request_data=body,
|
||||
api_base=endpoint_url,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ to AWS Bedrock's CountTokens API format and vice versa.
|
|||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, Literal
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -18,6 +18,14 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
|
|||
# max_tokens field; CountTokens only counts input, so it has no effect
|
||||
# on any generation.
|
||||
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024
|
||||
BedrockCountTokensInputType: TypeAlias = Literal["converse", "invokeModel"]
|
||||
|
||||
|
||||
def _invoke_model_max_tokens(thinking: JsonValue) -> int:
|
||||
budget: Final = thinking.get("budget_tokens") if isinstance(thinking, dict) else None
|
||||
if not isinstance(budget, int):
|
||||
return DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
|
||||
return max(DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, budget + 1)
|
||||
|
||||
|
||||
def _json_dict(value: JsonValue) -> dict[str, JsonValue]:
|
||||
|
|
@ -72,7 +80,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
- Response: {"inputTokens": <number>}
|
||||
"""
|
||||
|
||||
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> Literal["converse", "invokeModel"]:
|
||||
def _detect_input_type(self, request_data: Mapping[str, JsonValue]) -> BedrockCountTokensInputType:
|
||||
"""
|
||||
Detect whether to use 'converse' or 'invokeModel' input format.
|
||||
|
||||
|
|
@ -103,6 +111,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
def transform_anthropic_to_bedrock_count_tokens(
|
||||
self,
|
||||
request_data: Mapping[str, JsonValue],
|
||||
input_type: BedrockCountTokensInputType | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
Transform request to Bedrock CountTokens format.
|
||||
|
|
@ -133,9 +142,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
}
|
||||
}
|
||||
"""
|
||||
input_type: Final = self._detect_input_type(request_data)
|
||||
|
||||
if input_type == "converse":
|
||||
if (input_type or self._detect_input_type(request_data)) == "converse":
|
||||
return self._transform_to_converse_format(request_data)
|
||||
else:
|
||||
return self._transform_to_invoke_model_format(request_data)
|
||||
|
|
@ -196,7 +203,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
# Bedrock validates the body against the model's InvokeModel schema;
|
||||
# Anthropic Messages bodies require these fields.
|
||||
body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
|
||||
body_data.setdefault("max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS)
|
||||
body_data.setdefault("max_tokens", _invoke_model_max_tokens(body_data.get("thinking")))
|
||||
|
||||
# The CountTokens API expects invokeModel.body as a base64-encoded blob
|
||||
encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode()
|
||||
|
|
@ -257,23 +264,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
|
||||
return {"input_tokens": input_tokens}
|
||||
|
||||
def validate_count_tokens_request(self, request_data: Mapping[str, JsonValue]) -> None:
|
||||
def validate_count_tokens_request(
|
||||
self, request_data: Mapping[str, JsonValue], input_type: BedrockCountTokensInputType | None = None
|
||||
) -> None:
|
||||
"""
|
||||
Validate the incoming count tokens request.
|
||||
Supports both Converse and InvokeModel input formats.
|
||||
|
||||
Args:
|
||||
request_data: The request payload
|
||||
|
||||
Raises:
|
||||
ValueError: If the request is invalid
|
||||
"""
|
||||
if not request_data.get("model"):
|
||||
raise ValueError("model parameter is required")
|
||||
|
||||
input_type: Final = self._detect_input_type(request_data)
|
||||
|
||||
if input_type == "converse":
|
||||
if (input_type or self._detect_input_type(request_data)) == "converse":
|
||||
# Validate Converse format (messages-based)
|
||||
messages: Final = request_data.get("messages", [])
|
||||
if not messages:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
|
|
@ -77,6 +77,7 @@ else:
|
|||
|
||||
|
||||
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class AmazonAnthropicClaudeMessagesConfig(
|
||||
|
|
@ -100,6 +101,17 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
def beta_headers_provider(self) -> str:
|
||||
return self.custom_llm_provider or "bedrock"
|
||||
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return headers
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ())
|
||||
|
||||
def model_specific_anthropic_betas(self, model: str, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
|
||||
if bedrock_supports_per_turn_control(
|
||||
model, _MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None))
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
|
|||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.llms.bedrock.common_utils import build_mantle_messages_url
|
||||
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
|
|
@ -67,6 +68,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
|
|||
def should_filter_anthropic_beta_headers(self) -> bool:
|
||||
return False
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return BaseAnthropicMessagesConfig.sent_anthropic_betas(self, headers, request)
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import datetime
|
|||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -498,6 +499,8 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "openai",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ OpenAI Token Counter implementation using the Responses API /input_tokens endpoi
|
|||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import openai
|
||||
|
|
@ -96,6 +97,8 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "openai",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
Count tokens using OpenAI's Responses API /input_tokens endpoint.
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from litellm.types.llms.vertex_ai import (
|
|||
PartType,
|
||||
Schema,
|
||||
)
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.utils import supports_response_schema, supports_system_messages
|
||||
|
||||
GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
|
||||
|
|
@ -1259,8 +1259,6 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
self,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> bool:
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
return custom_llm_provider == LlmProviders.VERTEX_AI.value
|
||||
|
||||
async def count_tokens(
|
||||
|
|
@ -1275,6 +1273,8 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "openai",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
||||
|
|
@ -1313,6 +1313,24 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
|
||||
# Check if this is a partner model (Claude, Mistral, etc.)
|
||||
if VertexAIPartnerModels.is_vertex_partner_model(model_to_use):
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
ChatCountTranslationError,
|
||||
claude_count_options,
|
||||
)
|
||||
|
||||
sent_messages: Final = messages or []
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use,
|
||||
LlmProviders.VERTEX_AI,
|
||||
sent_messages,
|
||||
tools,
|
||||
request_options,
|
||||
anthropic_beta,
|
||||
deployment,
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
|
||||
|
||||
# Use partner models token counter
|
||||
partner_models_handler: Final = VertexAIPartnerModels()
|
||||
|
||||
|
|
@ -1336,13 +1354,19 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
try:
|
||||
result = await partner_models_handler.count_tokens(
|
||||
model=model_to_use,
|
||||
messages=messages or [],
|
||||
messages=sent_messages
|
||||
if claude_options.messages is None
|
||||
else [dict(message) for message in claude_options.messages],
|
||||
litellm_params=partner_litellm_params,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
system=system,
|
||||
tools=tools,
|
||||
tools=tools
|
||||
if claude_options.messages is None
|
||||
else [dict(tool) for tool in claude_options.tools or ()] or None,
|
||||
count_options=claude_options.options,
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
|
||||
status_code: Final = e.response.status_code
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
messages_carry_output_config,
|
||||
|
|
@ -17,6 +19,11 @@ from ....vertex_llm_base import VertexBase
|
|||
from ..output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
||||
|
||||
def _is_web_search(tool: Mapping[str, object]) -> bool:
|
||||
tool_type: Final = tool.get("type")
|
||||
return isinstance(tool_type, str) and tool_type.startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value)
|
||||
|
||||
|
||||
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
|
|
@ -66,59 +73,47 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
|
||||
headers["content-type"] = "application/json"
|
||||
|
||||
# Add beta headers for Vertex AI
|
||||
tools: Final = optional_params.get("tools", [])
|
||||
beta_values: Final[set[str]] = set()
|
||||
return dict(self.headers_with_anthropic_betas(headers, optional_params, messages)), api_base
|
||||
|
||||
# Get existing beta headers if any
|
||||
existing_beta: Final = headers.get("anthropic-beta")
|
||||
if existing_beta:
|
||||
beta_values.update(b.strip() for b in existing_beta.split(","))
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", [])
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
|
||||
# Check for web search tool
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value)
|
||||
break
|
||||
|
||||
# Check for tool search tools - Vertex AI uses different beta header
|
||||
anthropic_model_info: Final = AnthropicModelInfo()
|
||||
if anthropic_model_info.is_tool_search_used(tools):
|
||||
beta_values.add(get_tool_search_beta_header("vertex_ai"))
|
||||
|
||||
if optional_params.get("safeguards") is not None:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
|
||||
|
||||
if messages_carry_output_config(messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
|
||||
|
||||
if beta_values:
|
||||
headers["anthropic-beta"] = ",".join(beta_values)
|
||||
|
||||
return headers, api_base
|
||||
def headers_with_anthropic_betas(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
tools: Final = tuple(
|
||||
tool for tool in map(as_str_mapping, iter_items(optional_params.get("tools") or ())) if tool is not None
|
||||
)
|
||||
existing_betas: Final = (headers.get("anthropic-beta") or "").split(",")
|
||||
beta_values: Final = frozenset(
|
||||
(
|
||||
*(beta.strip() for beta in existing_betas if beta.strip()),
|
||||
*context_management_betas(optional_params.get("context_management")), # pyright: ignore[reportArgumentType] # optional_params is the config contract's untyped request body
|
||||
*(
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value,)
|
||||
if any(map(_is_web_search, tools))
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(get_tool_search_beta_header("vertex_ai"),)
|
||||
if AnthropicModelInfo().is_tool_search_used(list(tools))
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,)
|
||||
if optional_params.get("safeguards") is not None
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value,)
|
||||
if messages_carry_output_config(messages)
|
||||
else ()
|
||||
),
|
||||
)
|
||||
)
|
||||
if not beta_values:
|
||||
return headers
|
||||
return {**headers, "anthropic-beta": ",".join(sorted(beta_values))}
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Unlike Gemini models which use Google's token counting API, partner models use
|
|||
their respective publisher-specific count-tokens endpoints.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
|
@ -91,6 +92,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
model: str,
|
||||
request_data: dict[str, Any],
|
||||
litellm_params: dict[str, Any],
|
||||
betas: Sequence[str] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Handle token counting request for a Vertex AI partner model.
|
||||
|
|
@ -158,7 +160,10 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
)
|
||||
|
||||
# Prepare headers
|
||||
headers: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
headers: Final = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
**({"anthropic-beta": ",".join(betas)} if betas else {}),
|
||||
}
|
||||
|
||||
# Get async HTTP client
|
||||
from litellm import LlmProviders
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# What is this?
|
||||
## API Handler for calling Vertex AI Partner Models
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -266,6 +266,8 @@ class VertexAIPartnerModels(VertexBase):
|
|||
vertex_credentials=None,
|
||||
system: object | None = None,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
count_options: Mapping[str, object] = MappingProxyType({}),
|
||||
betas: Sequence[str] = (),
|
||||
):
|
||||
"""
|
||||
Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
|
||||
|
|
@ -302,6 +304,7 @@ class VertexAIPartnerModels(VertexBase):
|
|||
**MappingProxyType(
|
||||
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
|
||||
),
|
||||
**count_options,
|
||||
}
|
||||
|
||||
# Prepare litellm_params with credentials
|
||||
|
|
@ -319,6 +322,7 @@ class VertexAIPartnerModels(VertexBase):
|
|||
model=model,
|
||||
request_data=request_data,
|
||||
litellm_params=_litellm_params,
|
||||
betas=betas,
|
||||
)
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, should_auto_drop_params_for_agentic_cli
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
|
@ -333,12 +333,14 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
|
|||
|
||||
|
||||
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
|
||||
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format", "context_management")
|
||||
|
||||
|
||||
class _CountTokensBodyOptions(TypedDict):
|
||||
output_format: ReadOnly[JsonValue]
|
||||
context_management: ReadOnly[JsonValue]
|
||||
request_litellm_params: ReadOnly[Mapping[str, JsonValue]]
|
||||
request_options: ReadOnly[Mapping[str, JsonValue]]
|
||||
|
||||
|
||||
def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions:
|
||||
|
|
@ -349,6 +351,9 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
|
|||
"request_litellm_params": MappingProxyType(
|
||||
{"drop_params": True, **sent_params} if auto_drop_params else sent_params
|
||||
),
|
||||
"request_options": MappingProxyType(
|
||||
{key: data[key] for key in _COUNTED_REQUEST_OPTIONS if data.get(key) is not None}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -381,7 +386,7 @@ async def count_tokens(
|
|||
|
||||
Returns: {"input_tokens": <number>}
|
||||
"""
|
||||
from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config
|
||||
from litellm.proxy.proxy_server import count_request_tokens, general_settings_view, llm_router, proxy_config
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
|
||||
try:
|
||||
|
|
@ -412,6 +417,11 @@ async def count_tokens(
|
|||
request=token_request,
|
||||
call_endpoint=True,
|
||||
messages_format="anthropic",
|
||||
anthropic_beta=request.headers.get("anthropic-beta"),
|
||||
forward_client_headers=LiteLLMProxyRequestSetup.forwards_client_headers_for_all_models(
|
||||
general_settings_view()
|
||||
)
|
||||
or LiteLLMProxyRequestSetup.forwards_client_headers_for_model_group(token_request.model, user_api_key_dict),
|
||||
**_count_tokens_body_options(data, auto_drop_params),
|
||||
)
|
||||
except UnsupportedContentTypeError as e:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
|
|||
import litellm
|
||||
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.count_tokens.transformation import is_request_rejection
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
|
@ -66,6 +67,10 @@ class ProviderCountFailure:
|
|||
def cannot_count_model(self) -> bool:
|
||||
return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES
|
||||
|
||||
@property
|
||||
def rejected_request(self) -> bool:
|
||||
return is_request_rejection(self.status_code)
|
||||
|
||||
def with_hint(self, hint: str) -> "ProviderCountFailure":
|
||||
return replace(self, message=_with_hint(self.message, hint))
|
||||
|
||||
|
|
|
|||
|
|
@ -1527,27 +1527,33 @@ class LiteLLMProxyRequestSetup:
|
|||
return returned_headers
|
||||
|
||||
@staticmethod
|
||||
def add_headers_to_llm_call_by_model_group(data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth) -> dict:
|
||||
"""
|
||||
Add headers to the LLM call by model group
|
||||
"""
|
||||
def forwards_client_headers_for_all_models(general_settings: Mapping[str, object] | None) -> bool:
|
||||
return general_settings is not None and general_settings.get("forward_client_headers_to_llm_api") is True
|
||||
|
||||
@staticmethod
|
||||
def forwards_client_headers_for_model_group(model: str | None, user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
from litellm.proxy.auth.auth_checks import check_model_access_helper
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
data_model: Final = data.get("model")
|
||||
|
||||
if (
|
||||
data_model is not None
|
||||
return (
|
||||
model is not None
|
||||
and litellm.model_group_settings is not None
|
||||
and litellm.model_group_settings.forward_client_headers_to_llm_api is not None
|
||||
and check_model_access_helper(
|
||||
model=data_model,
|
||||
model=model,
|
||||
llm_router=llm_router,
|
||||
models=litellm.model_group_settings.forward_client_headers_to_llm_api,
|
||||
team_model_aliases=user_api_key_dict.team_model_aliases,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
) # handles aliases, wildcards, etc.
|
||||
):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def add_headers_to_llm_call_by_model_group(data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth) -> dict:
|
||||
"""
|
||||
Add headers to the LLM call by model group
|
||||
"""
|
||||
if LiteLLMProxyRequestSetup.forwards_client_headers_for_model_group(data.get("model"), user_api_key_dict):
|
||||
_headers: Final = LiteLLMProxyRequestSetup.add_headers_to_llm_call(headers, user_api_key_dict)
|
||||
if _headers != {}:
|
||||
data["headers"] = _headers
|
||||
|
|
@ -1584,7 +1590,7 @@ class LiteLLMProxyRequestSetup:
|
|||
"""
|
||||
data: Final = LitellmDataForBackendLLMCall()
|
||||
|
||||
if general_settings and general_settings.get("forward_client_headers_to_llm_api") is True:
|
||||
if LiteLLMProxyRequestSetup.forwards_client_headers_for_all_models(general_settings):
|
||||
_headers: Final = LiteLLMProxyRequestSetup.add_headers_to_llm_call(headers, user_api_key_dict)
|
||||
if _headers != {}:
|
||||
data["headers"] = _headers
|
||||
|
|
@ -4038,14 +4044,19 @@ async def add_guardrails_from_policy_engine(
|
|||
)
|
||||
|
||||
|
||||
_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
||||
(
|
||||
LlmProviders.ANTHROPIC.value,
|
||||
LlmProviders.BEDROCK.value,
|
||||
LlmProviders.BEDROCK_MANTLE.value,
|
||||
LlmProviders.VERTEX_AI.value,
|
||||
)
|
||||
_ANTHROPIC_API_HEADER_PROVIDER_NAMES: Final = (
|
||||
LlmProviders.ANTHROPIC.value,
|
||||
LlmProviders.BEDROCK.value,
|
||||
LlmProviders.BEDROCK_MANTLE.value,
|
||||
LlmProviders.VERTEX_AI.value,
|
||||
)
|
||||
_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(_ANTHROPIC_API_HEADER_PROVIDER_NAMES)
|
||||
|
||||
|
||||
def receives_anthropic_api_headers(custom_llm_provider: str | None) -> bool:
|
||||
return custom_llm_provider in _ANTHROPIC_API_HEADER_PROVIDER_NAMES
|
||||
|
||||
|
||||
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ def generate_feedback_box():
|
|||
import contextlib
|
||||
from collections import defaultdict
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from functools import lru_cache, partial
|
||||
from functools import lru_cache, partial, reduce
|
||||
|
||||
import litellm
|
||||
import litellm._redis
|
||||
|
|
@ -640,7 +640,11 @@ from litellm.proxy.list_api.common import (
|
|||
problem_response,
|
||||
request_validation_problem,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
LiteLLMProxyRequestSetup,
|
||||
add_litellm_data_to_request,
|
||||
receives_anthropic_api_headers,
|
||||
)
|
||||
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
|
||||
rust_control_plane_router,
|
||||
)
|
||||
|
|
@ -1316,6 +1320,8 @@ class _WebsearchInterceptionLitellmSettings(TypedDict, total=False):
|
|||
_LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody)
|
||||
_LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody)
|
||||
_ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig)
|
||||
_COUNT_REQUEST_OPTIONS: Final = TypeAdapter(Mapping[str, JsonValue])
|
||||
_DROPPED_PARAM_PATHS: Final = TypeAdapter(tuple[object, ...])
|
||||
_WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings)
|
||||
_UNTYPED_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
|
@ -14279,6 +14285,8 @@ async def _try_provider_token_count(
|
|||
messages_format: TokenCountMessagesFormat = "openai",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
|
||||
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
|
||||
|
|
@ -14305,6 +14313,8 @@ async def _try_provider_token_count(
|
|||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
request_options=request_options,
|
||||
anthropic_beta=anthropic_beta,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
return ProviderCountFailure(
|
||||
|
|
@ -14358,6 +14368,33 @@ async def _provider_token_count(
|
|||
raise failure.to_proxy_exception()
|
||||
|
||||
|
||||
def _retried_without_rejected_options(
|
||||
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
|
||||
request_options: Mapping[str, JsonValue],
|
||||
) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]:
|
||||
if not request_options:
|
||||
return count_with
|
||||
return partial(_count_retrying_without_options, count_with, tuple(request_options))
|
||||
|
||||
|
||||
async def _count_retrying_without_options(
|
||||
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
|
||||
forwarded_options: tuple[str, ...],
|
||||
model: str,
|
||||
region: str | None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
counted: Final = await count_with(model, region)
|
||||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
|
||||
return counted
|
||||
verbose_proxy_logger.warning(
|
||||
"Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",
|
||||
model,
|
||||
"could not be built" if counted.rejected_before_provider_call else f"answered {counted.status_code}",
|
||||
", ".join(forwarded_options),
|
||||
)
|
||||
return await count_with(model, region, request_options=MappingProxyType({}))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LocalTokenEstimate:
|
||||
total_tokens: int
|
||||
|
|
@ -14682,6 +14719,33 @@ def _count_context_management(
|
|||
)
|
||||
|
||||
|
||||
_STRING_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _deployment_with_request_drop_params(
|
||||
deployment: Mapping[str, object] | None, sent_params: Mapping[str, object]
|
||||
) -> Mapping[str, object] | None:
|
||||
if "drop_params" not in sent_params:
|
||||
return deployment
|
||||
resolved_deployment: Final = deployment or {}
|
||||
litellm_params: Final = _STRING_KEYED_MAPPING.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": sent_params["drop_params"]}}
|
||||
|
||||
|
||||
def _count_request_options(
|
||||
request_options: Mapping[str, JsonValue],
|
||||
deployment: Mapping[str, object] | None,
|
||||
sent_params: Mapping[str, object],
|
||||
) -> Mapping[str, JsonValue]:
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
|
||||
deployment_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
|
||||
dropped: Final = {**deployment_params, **sent_params}.get("additional_drop_params")
|
||||
dropped_values: Final = _DROPPED_PARAM_PATHS.validate_python(dropped) if isinstance(dropped, list) else ()
|
||||
dropped_paths: Final = tuple(path for path in dropped_values if isinstance(path, str))
|
||||
return _COUNT_REQUEST_OPTIONS.validate_python(reduce(delete_nested_value, dropped_paths, dict(request_options)))
|
||||
|
||||
|
||||
def _system_message(system: object) -> ChatCompletionSystemMessage | None:
|
||||
if not isinstance(system, (str, list)) or not system:
|
||||
return None
|
||||
|
|
@ -14714,6 +14778,9 @@ async def count_request_tokens(
|
|||
output_format: JsonValue = None,
|
||||
context_management: JsonValue = None,
|
||||
request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
forward_client_headers: bool = False,
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -14781,23 +14848,40 @@ async def count_request_tokens(
|
|||
prompt_only: Final = bool(prompt) and conversation_empty
|
||||
contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents))
|
||||
prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES
|
||||
sent_params: Final[Mapping[str, object]] = {
|
||||
**(_router_default_litellm_params(llm_router) if llm_router is not None else {}),
|
||||
**request_litellm_params,
|
||||
}
|
||||
counted_options: Final = _count_request_options(request_options, deployment_view, sent_params)
|
||||
ladder: Final = (
|
||||
None
|
||||
if provider_counter is None
|
||||
else await _provider_token_count(
|
||||
count_with=partial(
|
||||
_try_provider_token_count,
|
||||
provider_counter,
|
||||
custom_llm_provider,
|
||||
messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages,
|
||||
contents=None if prompt_as_message else contents,
|
||||
deployment=deployment,
|
||||
request_model=request.model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=_count_context_management(context_management, request.model, request_litellm_params),
|
||||
count_with=_retried_without_rejected_options(
|
||||
partial(
|
||||
_try_provider_token_count,
|
||||
provider_counter,
|
||||
custom_llm_provider,
|
||||
messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages,
|
||||
contents=None if prompt_as_message else contents,
|
||||
deployment=_deployment_with_request_drop_params(
|
||||
None if deployment is None else _STRING_KEYED_MAPPING.validate_python(deployment),
|
||||
sent_params,
|
||||
),
|
||||
request_model=request.model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=_count_context_management(
|
||||
context_management, request.model, request_litellm_params
|
||||
),
|
||||
request_options=counted_options,
|
||||
anthropic_beta=anthropic_beta
|
||||
if forward_client_headers or receives_anthropic_api_headers(custom_llm_provider)
|
||||
else None,
|
||||
),
|
||||
counted_options,
|
||||
),
|
||||
model_to_use=model_to_use,
|
||||
settings=settings,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
import asyncio
|
||||
from base64 import b64encode
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from queue import SimpleQueue
|
||||
from threading import get_ident
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -10,16 +13,25 @@ import respx
|
|||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
|
||||
AzureAIAnthropicCountTokensHandler,
|
||||
)
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_ADVISOR_TOOL_TYPE,
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -441,3 +453,337 @@ async def test_handler_sends_the_leading_system_run_as_system_not_as_a_message(h
|
|||
"system": [{"type": "text", "text": "Be terse"}],
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}],
|
||||
}
|
||||
|
||||
|
||||
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {"type": "object", "properties": {"city": {"type": "string"}}}
|
||||
_ANSWER_SCHEMA: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"answer": {"type": "string"}},
|
||||
"required": ["answer"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
|
||||
_COUNTERS: Final = pytest.mark.parametrize(
|
||||
("counter_type", "count_endpoint"),
|
||||
(
|
||||
(AnthropicTokenCounter, "https://gateway.example/v1/messages/count_tokens"),
|
||||
(AzureAIAnthropicTokenCounter, "https://gateway.example/anthropic/v1/messages/count_tokens"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
async def _counted_requests(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
messages: list[dict[str, JsonValue]],
|
||||
tools: list[dict[str, JsonValue]] | None,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
model: str = "claude-opus-5-5",
|
||||
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def count(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(200, json={"input_tokens": 31})
|
||||
|
||||
with respx.mock:
|
||||
respx.post(count_endpoint).mock(side_effect=count)
|
||||
result: Final = await counter_type().count_tokens(
|
||||
model_to_use=model,
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
request_model=model,
|
||||
tools=tools,
|
||||
messages_format=messages_format,
|
||||
request_options=request_options,
|
||||
anthropic_beta=anthropic_beta,
|
||||
)
|
||||
return result, tuple(sent.get() for _ in range(sent.qsize()))
|
||||
|
||||
|
||||
_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
_INFERENCE_PROVIDERS: Final = MappingProxyType(
|
||||
{AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"}
|
||||
)
|
||||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}
|
||||
_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000}
|
||||
|
||||
|
||||
async def _messages_inference_request(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
inference_endpoint: str,
|
||||
request_options: Mapping[str, JsonValue],
|
||||
model: str = "claude-haiku-5-5",
|
||||
extra_headers: Mapping[str, str] = MappingProxyType({}),
|
||||
) -> httpx.Request:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def reply(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(200, json=_MESSAGE_REPLY)
|
||||
|
||||
with respx.mock:
|
||||
respx.post(inference_endpoint).mock(side_effect=reply)
|
||||
await litellm.anthropic_messages(
|
||||
model=f"{_INFERENCE_PROVIDERS[counter_type]}/{model}",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
max_tokens=4096,
|
||||
api_key="sk-ant-api03-test-key",
|
||||
api_base="https://gateway.example",
|
||||
extra_headers=dict(extra_headers),
|
||||
**request_options,
|
||||
)
|
||||
return sent.get()
|
||||
|
||||
|
||||
def _on_both_counters(
|
||||
*betas: str,
|
||||
) -> Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]]:
|
||||
return MappingProxyType({AnthropicTokenCounter: frozenset(betas), AzureAIAnthropicTokenCounter: frozenset(betas)})
|
||||
|
||||
|
||||
def _betas(request: httpx.Request) -> frozenset[str]:
|
||||
return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
|
||||
@_COUNTERS
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "expected_betas"),
|
||||
(
|
||||
({"thinking": _ENABLED_THINKING}, _on_both_counters()),
|
||||
({"tool_choice": {"type": "any"}}, _on_both_counters()),
|
||||
(
|
||||
{"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}},
|
||||
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value),
|
||||
),
|
||||
(
|
||||
{
|
||||
"thinking": _ENABLED_THINKING,
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
},
|
||||
},
|
||||
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value),
|
||||
),
|
||||
(
|
||||
{"context_management": {"edits": [{"type": "compact_20260112"}]}},
|
||||
MappingProxyType(
|
||||
{
|
||||
AnthropicTokenCounter: frozenset((ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,)),
|
||||
AzureAIAnthropicTokenCounter: frozenset(),
|
||||
}
|
||||
),
|
||||
),
|
||||
),
|
||||
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
|
||||
)
|
||||
async def test_counter_sends_the_options_and_context_edits_messages_inference_sends(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
request_options: dict[str, JsonValue],
|
||||
expected_betas: Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]],
|
||||
) -> None:
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type, count_endpoint.removesuffix("/count_tokens"), request_options
|
||||
)
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
request_options=request_options,
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
inference_body: Final = _JSON_BODY.validate_json(inference.content)
|
||||
count_body: Final = _JSON_BODY.validate_json(count.content)
|
||||
inference_output_format: Final = inference_body.get("output_format")
|
||||
inference_output_config: Final = inference_body.get("output_config")
|
||||
assert "output_format" not in count_body
|
||||
assert count_body.get("output_config") == (
|
||||
inference_output_config
|
||||
if inference_output_format is None
|
||||
else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format}
|
||||
)
|
||||
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
|
||||
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
|
||||
}
|
||||
assert expected_betas[counter_type] == _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_sends_the_thinking_display_and_caller_betas_messages_inference_sends(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
caller_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
|
||||
display_thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type,
|
||||
count_endpoint.removesuffix("/count_tokens"),
|
||||
display_thinking,
|
||||
model="claude-opus-4-6",
|
||||
extra_headers={"anthropic-beta": caller_beta},
|
||||
)
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
request_options=display_thinking,
|
||||
anthropic_beta=caller_beta,
|
||||
model="claude-opus-4-6",
|
||||
)
|
||||
|
||||
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
|
||||
assert counted_betas == _betas(inference)
|
||||
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_without_options_sends_only_the_token_counting_beta_and_the_bare_count_body(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
anthropic_beta=ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value,
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
assert _betas(count) == {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
|
||||
assert _JSON_BODY.validate_json(count.content) == {
|
||||
"model": "claude-haiku-5-5",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"tools": [_WEATHER_TOOL],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_sending_options_still_counts_the_leading_system_run_as_system(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "system", "content": "Be terse."}, {"role": "user", "content": "Weather in SF?"}],
|
||||
None,
|
||||
"anthropic",
|
||||
request_options={"thinking": _ENABLED_THINKING},
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
body: Final = _JSON_BODY.validate_json(count.content)
|
||||
assert "thinking" in body
|
||||
assert {name: body.get(name) for name in ("system", "messages")} == {
|
||||
"system": [{"type": "text", "text": "Be terse."}],
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
}
|
||||
|
||||
|
||||
_ADVISOR_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": ANTHROPIC_ADVISOR_TOOL_TYPE,
|
||||
"name": "advisor",
|
||||
"model": "claude-opus-5-5",
|
||||
}
|
||||
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}}
|
||||
_ADVISOR_RESULT: Final[dict[str, JsonValue]] = {
|
||||
"type": "advisor_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": {"type": "advisor_result", "text": "Check the forecast first."},
|
||||
}
|
||||
_ADVISED_ANSWER: Final[dict[str, JsonValue]] = {"type": "text", "text": "It is sunny."}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "counted_reply"),
|
||||
[
|
||||
pytest.param(
|
||||
[_ADVISOR_TOOL, _WEATHER_TOOL],
|
||||
[_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER],
|
||||
id="advisor_tool_sent",
|
||||
),
|
||||
pytest.param([_WEATHER_TOOL], [_ADVISED_ANSWER], id="advisor_tool_not_sent"),
|
||||
],
|
||||
)
|
||||
async def test_counter_sending_options_keeps_advisor_blocks_only_when_the_advisor_tool_is_sent(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
tools: list[dict[str, JsonValue]],
|
||||
counted_reply: list[JsonValue],
|
||||
) -> None:
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[
|
||||
{"role": "user", "content": "Weather in SF?"},
|
||||
{"role": "assistant", "content": [_ADVISOR_CALL, _ADVISOR_RESULT, _ADVISED_ANSWER]},
|
||||
{"role": "user", "content": "And tomorrow?"},
|
||||
],
|
||||
tools,
|
||||
"anthropic",
|
||||
request_options={"thinking": _ENABLED_THINKING},
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
body: Final = _JSON_BODY.validate_json(count.content)
|
||||
assert {name: body.get(name) for name in ("messages", "tools")} == {
|
||||
"messages": [
|
||||
{"role": "user", "content": "Weather in SF?"},
|
||||
{"role": "assistant", "content": counted_reply},
|
||||
{"role": "user", "content": "And tomorrow?"},
|
||||
],
|
||||
"tools": tools,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_refuses_options_messages_inference_refuses_without_calling_anthropic(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
result, requests = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
None,
|
||||
"anthropic",
|
||||
request_options={"output_config": {"effort": "fastest"}},
|
||||
model="claude-haiku-4-5",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
assert requests == ()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import base64
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.bedrock.count_tokens.transformation import (
|
||||
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS,
|
||||
|
|
@ -53,6 +56,28 @@ def test_detect_input_type_converse_blocks_route_to_converse():
|
|||
assert config._detect_input_type(request) == "converse"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"option",
|
||||
(
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
{"tool_choice": {"type": "auto"}},
|
||||
{"output_config": {"effort": "low"}},
|
||||
),
|
||||
)
|
||||
def test_pass_through_body_with_string_content_is_counted_as_converse_whatever_options_it_carries(
|
||||
option: dict[str, object],
|
||||
) -> None:
|
||||
request: Final = {
|
||||
"model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
**option,
|
||||
}
|
||||
|
||||
result: Final = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request)
|
||||
|
||||
assert result["input"] == {"converse": {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}}
|
||||
|
||||
|
||||
def test_transform_to_invoke_model_format_base64_encodes_body():
|
||||
"""The CountTokens API expects invokeModel.body as a base64-encoded blob;
|
||||
Anthropic Messages bodies additionally need anthropic_version/max_tokens
|
||||
|
|
|
|||
|
|
@ -1,12 +1,20 @@
|
|||
from collections.abc import Mapping
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_EFFORT_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
|
||||
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
|
|
@ -251,3 +259,500 @@ async def test_explicit_mantle_failure_does_not_probe_runtime() -> None:
|
|||
assert result.status_code == 404
|
||||
assert result.tokenizer_type == "bedrock_mantle_api"
|
||||
assert bedrock.posted_hosts() == (MANTLE_HOST,)
|
||||
|
||||
|
||||
def _invoke_body(request: httpx.Request) -> dict[str, JsonValue]:
|
||||
invoke_model: Final = _JSON_BODY.validate_json(request.content)["input"]
|
||||
assert isinstance(invoke_model, dict)
|
||||
encoded: Final = invoke_model["invokeModel"]
|
||||
assert isinstance(encoded, dict) and isinstance(encoded["body"], str)
|
||||
return _JSON_BODY.validate_json(base64.b64decode(encoded["body"]))
|
||||
|
||||
|
||||
_CLAUDE_REPLY: Final = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
}
|
||||
_THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
"tool_choice": {"type": "auto"},
|
||||
}
|
||||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
|
||||
|
||||
|
||||
_CITY_SCHEMA: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}
|
||||
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
|
||||
|
||||
|
||||
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
|
||||
|
||||
|
||||
async def _messages_inference_and_count(
|
||||
deployment_model: str,
|
||||
model_to_use: str,
|
||||
request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS,
|
||||
tool: Mapping[str, JsonValue] = _WEATHER_TOOL,
|
||||
runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}),
|
||||
inference_reply: Mapping[str, object] = _CLAUDE_REPLY,
|
||||
) -> tuple[_Bedrock, _Bedrock]:
|
||||
credentials: Final = DEPLOYMENT["litellm_params"]
|
||||
inference: Final = _Bedrock(runtime=(200, inference_reply), mantle=(200, inference_reply))
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=MESSAGES,
|
||||
tools=[dict(tool)],
|
||||
max_tokens=4096,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
|
||||
**request_options,
|
||||
**credentials,
|
||||
)
|
||||
bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177}))
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use=model_to_use,
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {**credentials, "model": deployment_model}},
|
||||
request_model="claude",
|
||||
tools=[dict(tool)],
|
||||
request_options=request_options,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.error is False
|
||||
return inference, bedrock
|
||||
|
||||
|
||||
def _betas_besides_effort(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]:
|
||||
betas: Final = body.get("anthropic_beta") or []
|
||||
assert isinstance(betas, list)
|
||||
return tuple(beta for beta in betas if beta != ANTHROPIC_EFFORT_BETA_HEADER)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "model_to_use"),
|
||||
(
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-sonnet-4-6"),
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"),
|
||||
),
|
||||
)
|
||||
async def test_runtime_count_sends_the_options_and_context_edits_invoke_inference_sends(
|
||||
deployment_model: str, model_to_use: str
|
||||
) -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(deployment_model, model_to_use)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted: Final = _invoke_body(bedrock.requests[0])
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
|
||||
assert _betas_besides_effort(counted) == _betas_besides_effort(sent)
|
||||
thinking: Final = counted["thinking"]
|
||||
assert isinstance(thinking, dict)
|
||||
budget_tokens: Final = thinking.get("budget_tokens", 0)
|
||||
assert isinstance(budget_tokens, int) and isinstance(counted["max_tokens"], int)
|
||||
assert counted["max_tokens"] > budget_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5"
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (MANTLE_HOST,)
|
||||
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
|
||||
assert "anthropic_beta" not in counted
|
||||
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
|
||||
|
||||
|
||||
_DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive", "display": "updates"}}
|
||||
|
||||
|
||||
def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
|
||||
return tuple(sorted(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "request_options", "expected_beta"),
|
||||
(
|
||||
(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
|
||||
{"output_config": {"effort": "low"}},
|
||||
ANTHROPIC_EFFORT_BETA_HEADER,
|
||||
),
|
||||
("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER),
|
||||
),
|
||||
ids=("effort", "thinking_display"),
|
||||
)
|
||||
async def test_runtime_count_sends_the_betas_invoke_inference_sends(
|
||||
deployment_model: str, request_options: dict[str, JsonValue], expected_beta: str
|
||||
) -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta")
|
||||
assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
|
||||
assert isinstance(counted_betas, list) and expected_beta in counted_betas
|
||||
|
||||
|
||||
_EFFORT: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}}
|
||||
_CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = {
|
||||
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "expected_betas"),
|
||||
((_DISPLAY_THINKING, (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)), (_EFFORT, ())),
|
||||
ids=("thinking_display", "effort"),
|
||||
)
|
||||
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends(
|
||||
request_options: dict[str, JsonValue], expected_betas: tuple[str, ...]
|
||||
) -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
||||
"us.anthropic.claude-opus-5-5",
|
||||
request_options,
|
||||
runtime=(400, UNSUPPORTED),
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
|
||||
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", [])
|
||||
assert isinstance(sent_betas, list)
|
||||
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) == expected_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "request_options"),
|
||||
(
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _EFFORT),
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLEAR_THINKING_EDITS),
|
||||
("bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", _EFFORT),
|
||||
),
|
||||
ids=("effort", "clear_thinking_sets_effort", "inference_sends_the_effort_beta"),
|
||||
)
|
||||
async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_betas_invoke_inference_sends(
|
||||
deployment_model: str, request_options: dict[str, JsonValue]
|
||||
) -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted: Final = _invoke_body(bedrock.requests[0])
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
counted_config: Final = counted.get("output_config")
|
||||
assert isinstance(counted_config, dict) and counted_config.get("effort") == "low"
|
||||
counted_betas: Final = counted.get("anthropic_beta")
|
||||
sent_betas: Final = sent.get("anthropic_beta", [])
|
||||
assert isinstance(counted_betas, list) and isinstance(sent_betas, list)
|
||||
assert counted_betas.count(ANTHROPIC_EFFORT_BETA_HEADER) == 1
|
||||
assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock/invoke/us.anthropic.claude-sonnet-4-6",
|
||||
"us.anthropic.claude-sonnet-4-6",
|
||||
{**_CITY_FORMAT, "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted: Final = _invoke_body(bedrock.requests[0])
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
assert counted.get("output_config") == sent.get("output_config")
|
||||
assert counted.get("anthropic_beta") == sent.get("anthropic_beta")
|
||||
|
||||
|
||||
_CONVERSE_REPLY: Final = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
_NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = {
|
||||
"tool_choice": {"type": "auto", "disable_parallel_tool_use": True}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "inference_reply"),
|
||||
(
|
||||
("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY),
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY),
|
||||
),
|
||||
ids=("converse", "invoke"),
|
||||
)
|
||||
async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it(
|
||||
deployment_model: str, inference_reply: Mapping[str, object]
|
||||
) -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
deployment_model,
|
||||
deployment_model.split("/")[-1],
|
||||
_NO_PARALLEL_TOOL_USE,
|
||||
inference_reply=inference_reply,
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted_tool_choice: Final = _invoke_body(bedrock.requests[0]).get("tool_choice")
|
||||
assert isinstance(counted_tool_choice, dict) and counted_tool_choice.get("type") == "auto"
|
||||
assert ("disable_parallel_tool_use" in counted_tool_choice) == (
|
||||
b"disable_parallel_tool_use" in inference.requests[0].content
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_on_a_stand_in_model_counts_the_messages_and_output_config_the_deployment_model_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock/us.anthropic.claude-haiku-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _CITY_FORMAT
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (MANTLE_HOST,)
|
||||
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
fields: Final = ("messages", "output_config")
|
||||
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (MANTLE_HOST,)
|
||||
assert _sent_betas(bedrock.requests[0]) == _sent_betas(inference.requests[0])
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0])
|
||||
|
||||
|
||||
def _mantle_answers(status_code: int, body: Mapping[str, JsonValue]) -> Callable[[httpx.Request], httpx.Response]:
|
||||
return lambda request: httpx.Response(status_code, json=dict(body), request=request)
|
||||
|
||||
|
||||
def _mantle_unreachable(error: type[httpx.TransportError]) -> Callable[[httpx.Request], httpx.Response]:
|
||||
def unreachable(request: httpx.Request) -> httpx.Response:
|
||||
raise error("bedrock-mantle is unreachable", request=request)
|
||||
|
||||
return unreachable
|
||||
|
||||
|
||||
_MANTLE_REJECTION: Final[dict[str, JsonValue]] = {
|
||||
"type": "error",
|
||||
"error": {"type": "invalid_request_error", "message": "thinking: unsupported"},
|
||||
}
|
||||
_MANTLE_FAILURE: Final[dict[str, JsonValue]] = {"type": "error", "error": {"type": "api_error", "message": "down"}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "mantle", "expected"),
|
||||
(
|
||||
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ConnectError), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ReadTimeout), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(503, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(429, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(403, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(400, _MANTLE_REJECTION), (True, 400, "bedrock_mantle_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(422, _MANTLE_REJECTION), (True, 422, "bedrock_mantle_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(200, {"input_tokens": 2177}), (False, None, "bedrock_mantle_api")),
|
||||
({}, _mantle_answers(503, _MANTLE_FAILURE), (True, 503, "bedrock_mantle_api")),
|
||||
),
|
||||
ids=(
|
||||
"mantle-unreachable",
|
||||
"mantle-timeout",
|
||||
"mantle-5xx",
|
||||
"mantle-rate-limit",
|
||||
"mantle-denied",
|
||||
"mantle-rejects-the-options",
|
||||
"mantle-rejects-the-body",
|
||||
"mantle-counts",
|
||||
"no-options-mantle-5xx",
|
||||
),
|
||||
)
|
||||
async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or_rejects_the_request(
|
||||
request_options: dict[str, JsonValue],
|
||||
mantle: Callable[[httpx.Request], httpx.Response],
|
||||
expected: tuple[bool, int | None, str],
|
||||
) -> None:
|
||||
def bedrock(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == RUNTIME_HOST:
|
||||
return httpx.Response(400, json=UNSUPPORTED, request=request)
|
||||
return mantle(request)
|
||||
|
||||
result: Final = await BedrockTokenCounter(
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock))
|
||||
).count_tokens(
|
||||
model_to_use="us.anthropic.claude-opus-5-5",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={
|
||||
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"}
|
||||
},
|
||||
request_model="claude-opus-5-5",
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.tokenizer_type) == expected
|
||||
|
||||
|
||||
_CALLER_BETAS: Final = (ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value, "context-1m-2025-08-07")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_count_sends_the_caller_betas_invoke_inference_keeps_for_the_model() -> None:
|
||||
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0"
|
||||
model_info: Final[dict[str, JsonValue]] = {"supports_per_turn_control": True}
|
||||
thinking: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}}
|
||||
credentials: Final = DEPLOYMENT["litellm_params"]
|
||||
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=MESSAGES,
|
||||
max_tokens=4096,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
|
||||
extra_headers={"anthropic-beta": ",".join(_CALLER_BETAS)},
|
||||
model_info=model_info,
|
||||
**thinking,
|
||||
**credentials,
|
||||
)
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use=deployment_model.removeprefix("bedrock/invoke/"),
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
|
||||
request_model="claude",
|
||||
request_options=thinking,
|
||||
anthropic_beta=",".join(_CALLER_BETAS),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.error is False
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted_betas: Final = _invoke_body(bedrock.requests[0]).get("anthropic_beta")
|
||||
assert counted_betas == _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta")
|
||||
assert isinstance(counted_betas, list) and set(_CALLER_BETAS) <= set(counted_betas)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
||||
"us.anthropic.claude-opus-5-5",
|
||||
_THINKING_AND_TOOL_CHOICE,
|
||||
tool={**_WEATHER_TOOL, "eager_input_streaming": True},
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
|
||||
|
||||
|
||||
def _texts(message: JsonValue) -> tuple[str, ...]:
|
||||
content: Final = message.get("content") if isinstance(message, dict) else None
|
||||
if not isinstance(content, list):
|
||||
return ()
|
||||
return tuple(text for block in content if isinstance(block, dict) and isinstance(text := block.get("text"), str))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("native_output", "request_options", "schema_inlined"),
|
||||
(
|
||||
(False, _CITY_FORMAT, True),
|
||||
(True, _CITY_FORMAT, False),
|
||||
(False, _THINKING_AND_TOOL_CHOICE, False),
|
||||
),
|
||||
ids=("inlined_schema", "native_schema", "no_format"),
|
||||
)
|
||||
async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends(
|
||||
native_output: bool, request_options: dict[str, JsonValue], schema_inlined: bool
|
||||
) -> None:
|
||||
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-count-probe-v1:0"
|
||||
model_info: Final = {"supports_bedrock_native_structured_output": native_output}
|
||||
credentials: Final = DEPLOYMENT["litellm_params"]
|
||||
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=MESSAGES,
|
||||
max_tokens=4096,
|
||||
model_info=model_info,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
|
||||
**request_options,
|
||||
**credentials,
|
||||
)
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use="us.anthropic.claude-count-probe-v1:0",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
|
||||
request_model="claude",
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.total_tokens) == (False, 412)
|
||||
counted: Final = _invoke_body(bedrock.requests[0])["messages"]
|
||||
assert counted == _JSON_BODY.validate_json(inference.requests[0].content)["messages"]
|
||||
assert isinstance(counted, list)
|
||||
assert (json.dumps(_CITY_SCHEMA) in _texts(counted[-1])) is schema_inlined
|
||||
assert (counted == MESSAGES) is not schema_inlined
|
||||
|
||||
|
||||
_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "tools", "request_options"),
|
||||
(
|
||||
(MESSAGES, None, {"output_config": {"effort": "fastest"}}),
|
||||
(MESSAGES, [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], _THINKING),
|
||||
(MESSAGES, ["get_weather"], _THINKING),
|
||||
([{"role": "user", "content": 5}], None, _THINKING),
|
||||
([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING),
|
||||
),
|
||||
ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"),
|
||||
)
|
||||
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock(
|
||||
messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue]
|
||||
) -> None:
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment=DEPLOYMENT,
|
||||
request_model="claude-sonnet-4-5",
|
||||
tools=tools,
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
assert bedrock.posted_hosts() == ()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import copy
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from queue import SimpleQueue
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
|
@ -28,6 +30,7 @@ from litellm.llms.vertex_ai.common_utils import (
|
|||
vertex_request_labels_from_litellm_params,
|
||||
)
|
||||
from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
|
||||
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
|
||||
|
||||
|
||||
|
|
@ -2291,3 +2294,179 @@ async def test_vertex_ai_token_counter_routes_like_generate_content(
|
|||
)
|
||||
|
||||
assert (None if result is None else result.total_tokens) == vertex_counted_tokens
|
||||
|
||||
|
||||
_PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [
|
||||
{"role": "user", "content": "Weather in SF?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
_PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = {
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
|
||||
|
||||
async def _partner_counted_bodies(
|
||||
model: str, request_options: Mapping[str, JsonValue] = MappingProxyType({})
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def json(self) -> dict[str, int]:
|
||||
return {"input_tokens": 37}
|
||||
|
||||
class FakeHttpClient:
|
||||
posted_bodies: tuple[dict[str, object], ...] = ()
|
||||
|
||||
async def post(
|
||||
self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float
|
||||
) -> FakeResponse:
|
||||
self.posted_bodies = (*self.posted_bodies, json)
|
||||
return FakeResponse()
|
||||
|
||||
fake_http_client: Final = FakeHttpClient()
|
||||
with (
|
||||
patch.object(handler, "get_async_httpx_client", return_value=fake_http_client),
|
||||
patch.object(
|
||||
handler.VertexAIPartnerModelsTokenCounter,
|
||||
"_ensure_access_token_async",
|
||||
new=AsyncMock(return_value=("fake-token", "test-project")),
|
||||
),
|
||||
):
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use=model,
|
||||
messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS),
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=[_PARTNER_RESPONSES_TOOL],
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 37
|
||||
return fake_http_client.posted_bodies
|
||||
|
||||
|
||||
_PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
async def _vertex_claude_inference_and_count(
|
||||
tools: list[dict[str, JsonValue]], request_options: Mapping[str, JsonValue]
|
||||
) -> tuple[httpx.Request, httpx.Request]:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
||||
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def vertex(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(
|
||||
200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY
|
||||
)
|
||||
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex))
|
||||
with (
|
||||
patch.object(handler, "get_async_httpx_client", return_value=client),
|
||||
patch.object(
|
||||
VertexBase, "_ensure_access_token_async", new=AsyncMock(return_value=("fake-token", "test-project"))
|
||||
),
|
||||
patch.object(VertexBase, "_ensure_access_token", return_value=("fake-token", "test-project")),
|
||||
):
|
||||
await litellm.anthropic_messages(
|
||||
model="vertex_ai/claude-haiku-5-5",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
max_tokens=4096,
|
||||
vertex_project="test-project",
|
||||
vertex_location="us-east5",
|
||||
client=client,
|
||||
**request_options,
|
||||
)
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=tools,
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 37
|
||||
inference, count = tuple(sent.get() for _ in range(sent.qsize()))
|
||||
assert count.url.path.endswith("count-tokens:rawPredict")
|
||||
return inference, count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
|
||||
inference, count = await _vertex_claude_inference_and_count(
|
||||
[{"name": "get_weather", "input_schema": {"type": "object"}}],
|
||||
{
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
"tool_choice": {"type": "auto"},
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content)
|
||||
inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
assert {name: counted.get(name) for name in fields} == {name: inferred.get(name) for name in fields}
|
||||
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_sends_the_tool_search_beta_vertex_messages_inference_sends() -> None:
|
||||
inference, count = await _vertex_claude_inference_and_count(
|
||||
[
|
||||
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
|
||||
{"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True},
|
||||
],
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
)
|
||||
|
||||
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
|
||||
assert get_tool_search_beta_header("vertex_ai") in count.headers["anthropic-beta"].split(",")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None:
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use="claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
request_options={"output_config": {"effort": "fastest"}},
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_llama_count_leaves_out_the_claude_only_options() -> None:
|
||||
model: Final = "meta/llama-4-maverick-17b-128e-instruct-maas"
|
||||
|
||||
assert await _partner_counted_bodies(
|
||||
model, request_options={"thinking": {"type": "enabled", "budget_tokens": 2000}}
|
||||
) == await _partner_counted_bodies(model)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import anyio
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
import tiktoken
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue
|
||||
|
|
@ -36,6 +37,7 @@ load_dotenv()
|
|||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.types import Message
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -53,10 +55,11 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
|||
kimi_k3_tokens,
|
||||
load_kimi_k3_encoding,
|
||||
)
|
||||
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensInputType
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter
|
||||
from litellm.proxy._types import ProxyException, TokenCountRequest
|
||||
from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import (
|
||||
count_tokens as anthropic_count_tokens,
|
||||
)
|
||||
|
|
@ -76,6 +79,8 @@ from litellm.proxy.proxy_server import (
|
|||
token_counter,
|
||||
)
|
||||
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.router import CustomRoutingStrategyBase
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
|
||||
from litellm.types.utils import SelectTokenizerResponse, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
|
@ -240,6 +245,9 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
forward_client_headers=False,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "claude-3-sonnet-20240229"
|
||||
|
|
@ -320,6 +328,9 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
forward_client_headers=False,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "gpt-4"
|
||||
|
|
@ -809,6 +820,7 @@ class _FailingRuntimeHandler(BedrockCountTokensHandler):
|
|||
litellm_params: dict[str, object],
|
||||
resolved_model: str,
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
input_type: BedrockCountTokensInputType | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
raise self._error
|
||||
|
||||
|
|
@ -1138,6 +1150,9 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
forward_client_headers=False,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"detail":{"message":"Input is too long for requested model."}}',
|
||||
|
|
@ -1195,6 +1210,9 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
forward_client_headers=False,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"Message":"Bearer Token has expired"}',
|
||||
|
|
@ -1252,6 +1270,9 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
request_options=None,
|
||||
anthropic_beta=None,
|
||||
forward_client_headers=False,
|
||||
):
|
||||
raise ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
|
|
@ -1276,6 +1297,660 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
proxy_server.count_request_tokens = original_token_counter
|
||||
|
||||
|
||||
def _json_request(
|
||||
body: Mapping[str, object], user_agent: str, headers: Mapping[str, str] = MappingProxyType({})
|
||||
) -> Request:
|
||||
payload: Final = json.dumps(body).encode()
|
||||
|
||||
async def receive() -> Message:
|
||||
return {"type": "http.request", "body": payload, "more_body": False}
|
||||
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages/count_tokens",
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"user-agent", user_agent.encode()),
|
||||
*((name.encode(), value.encode()) for name, value in headers.items()),
|
||||
],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
|
||||
|
||||
_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "adaptive"},
|
||||
"tool_choice": {"type": "auto"},
|
||||
"output_config": {
|
||||
"effort": "low",
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
}
|
||||
|
||||
|
||||
def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
|
||||
output_config: Final = body.get("output_config")
|
||||
return frozenset(name for name in _CLAUDE_COUNT_OPTIONS if name in body) | frozenset(
|
||||
f"output_config.{key}" for key in (output_config if isinstance(output_config, dict) else {})
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("additional_drop_params", "request_params", "expected_paths"),
|
||||
(
|
||||
(
|
||||
None,
|
||||
{},
|
||||
{
|
||||
"thinking",
|
||||
"tool_choice",
|
||||
"output_config",
|
||||
"output_config.effort",
|
||||
"output_config.format",
|
||||
"context_management",
|
||||
},
|
||||
),
|
||||
(
|
||||
["tool_choice", "output_config.format"],
|
||||
{},
|
||||
{"thinking", "output_config", "output_config.effort", "context_management"},
|
||||
),
|
||||
(
|
||||
["context_management"],
|
||||
{},
|
||||
{"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"},
|
||||
),
|
||||
(
|
||||
None,
|
||||
{"additional_drop_params": ["thinking", "context_management"]},
|
||||
{"tool_choice", "output_config", "output_config.effort", "output_config.format"},
|
||||
),
|
||||
),
|
||||
ids=(
|
||||
"nothing-dropped",
|
||||
"deployment-drops-tool-choice-and-the-format",
|
||||
"deployment-drops-context-management",
|
||||
"request-drops-thinking-and-context-management",
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
additional_drop_params: list[str] | None,
|
||||
request_params: dict[str, JsonValue],
|
||||
expected_paths: set[str],
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-haiku-5-5",
|
||||
"api_key": "sk-ant-test",
|
||||
"additional_drop_params": additional_drop_params,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
request: Final = _json_request(
|
||||
{
|
||||
"model": "claude-agent",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
|
||||
**_CLAUDE_COUNT_OPTIONS,
|
||||
**request_params,
|
||||
},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 7})
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
|
||||
assert response["input_tokens"] == 7
|
||||
sent_request: Final = route.calls.last.request
|
||||
assert _sent_option_paths(json.loads(sent_request.content)) == expected_paths
|
||||
assert ("context-management-2025-06-27" in sent_request.headers["anthropic-beta"]) == (
|
||||
"context_management" in expected_paths
|
||||
)
|
||||
|
||||
|
||||
_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
|
||||
|
||||
|
||||
def _claude_router(deployment_model: str, deployment_params: Mapping[str, JsonValue]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {"model": deployment_model, "api_key": "sk-ant-test", **deployment_params},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def _count_on_a_claude_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_model: str,
|
||||
body: Mapping[str, JsonValue],
|
||||
reply: Callable[[httpx.Request], httpx.Response],
|
||||
user_agent: str = "python-httpx/0.28.1",
|
||||
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
router: Final = _claude_router(deployment_model, deployment_params)
|
||||
request: Final = _json_request(
|
||||
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
||||
user_agent,
|
||||
{"anthropic-beta": _CALLER_BETA},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(side_effect=reply)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
return response, tuple(call.request for call in route.calls)
|
||||
|
||||
|
||||
def _counts_only_without_options(request: httpx.Request) -> httpx.Response:
|
||||
if _sent_option_paths(json.loads(request.content)):
|
||||
return httpx.Response(
|
||||
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}}
|
||||
)
|
||||
return httpx.Response(200, json={"input_tokens": 7})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "options", "rejected_by_anthropic"),
|
||||
(
|
||||
("anthropic/claude-haiku-4-5", {"output_config": {"effort": "fastest"}}, False),
|
||||
("anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, True),
|
||||
),
|
||||
ids=("rejected-by-the-count-transform", "rejected-by-anthropic"),
|
||||
)
|
||||
async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_claude_count_rejects_them(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_model: str,
|
||||
options: dict[str, JsonValue],
|
||||
rejected_by_anthropic: bool,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch, deployment_model, options, _counts_only_without_options
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert response["input_tokens"] == 7
|
||||
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
|
||||
*((True,) if rejected_by_anthropic else ()),
|
||||
False,
|
||||
]
|
||||
assert [_CALLER_BETA in sent.headers.get("anthropic-beta", "") for sent in requests] == [
|
||||
*((True,) if rejected_by_anthropic else ()),
|
||||
False,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("status_code", "counted_again"), ((401, False), (403, False), (404, True), (422, True)))
|
||||
async def test_anthropic_count_tokens_counts_again_only_when_the_claude_count_rejects_the_request(
|
||||
monkeypatch: pytest.MonkeyPatch, status_code: int, counted_again: bool
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-haiku-5-5",
|
||||
_CLAUDE_COUNT_OPTIONS,
|
||||
lambda _: httpx.Response(status_code, json={"type": "error", "error": {"type": "error", "message": "denied"}}),
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == status_code
|
||||
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
|
||||
True,
|
||||
*((False,) if counted_again else ()),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_warns_with_every_option_it_counts_again_without(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
verbose_proxy_logger.addHandler(caplog.handler)
|
||||
try:
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
|
||||
await _count_on_a_claude_deployment(
|
||||
monkeypatch, "anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, _counts_only_without_options
|
||||
)
|
||||
finally:
|
||||
verbose_proxy_logger.removeHandler(caplog.handler)
|
||||
|
||||
warnings: Final = {
|
||||
record.getMessage() for record in caplog.records if "counts again without" in record.getMessage()
|
||||
}
|
||||
(warning,) = warnings
|
||||
assert all(option in warning for option in _CLAUDE_COUNT_OPTIONS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-haiku-5-5",
|
||||
_CLAUDE_COUNT_OPTIONS,
|
||||
lambda _: httpx.Response(429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow"}}),
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 429
|
||||
assert len(requests) == 1
|
||||
|
||||
|
||||
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
|
||||
"model": "bedrock/us.anthropic.claude-count-probe-v1:0",
|
||||
"aws_access_key_id": "AKIATESTACCESSKEY",
|
||||
"aws_secret_access_key": "test-secret",
|
||||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
|
||||
|
||||
async def _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
body: Mapping[str, JsonValue],
|
||||
runtime: Callable[[httpx.Request], httpx.Response],
|
||||
mantle: Callable[[httpx.Request], httpx.Response],
|
||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}])
|
||||
request: Final = _json_request(
|
||||
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock as bedrock:
|
||||
bedrock.route(host=_BEDROCK_RUNTIME_HOST).mock(side_effect=runtime)
|
||||
bedrock.route(host=_BEDROCK_MANTLE_HOST).mock(side_effect=mantle)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
return response, tuple(call.request for call in bedrock.calls)
|
||||
|
||||
|
||||
def _runtime_counts_only_without_options(request: httpx.Request) -> httpx.Response:
|
||||
if "invokeModel" in json.loads(request.content)["input"]:
|
||||
return httpx.Response(400, json={"message": "thinking: unsupported"})
|
||||
return httpx.Response(200, json={"inputTokens": 9})
|
||||
|
||||
|
||||
def _mantle_unreachable(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("bedrock-mantle is unreachable", request=request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"malformed",
|
||||
(
|
||||
{"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}]},
|
||||
{"messages": [{"role": "user", "content": 5}]},
|
||||
{"messages": [{"role": "user", "content": [{"text": "Weather in SF?"}]}]},
|
||||
),
|
||||
ids=("tool_type_not_a_string", "content_not_iterable", "block_without_type"),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_request_again_without_the_options(
|
||||
monkeypatch: pytest.MonkeyPatch, malformed: dict[str, JsonValue]
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch,
|
||||
{**malformed, "thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
_runtime_counts_only_without_options,
|
||||
_mantle_unreachable,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert response["input_tokens"] == 9
|
||||
assert [request.url.host for request in requests] == [_BEDROCK_RUNTIME_HOST]
|
||||
assert "invokeModel" not in json.loads(requests[0].content)["input"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_options_when_mantle_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch,
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
_runtime_counts_only_without_options,
|
||||
_mantle_unreachable,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert response["input_tokens"] == 9
|
||||
hosts: Final = [request.url.host for request in requests]
|
||||
assert hosts[0] == hosts[-1] == _BEDROCK_RUNTIME_HOST
|
||||
assert _BEDROCK_MANTLE_HOST in hosts
|
||||
assert "invokeModel" not in json.loads(requests[-1].content)["input"]
|
||||
|
||||
|
||||
_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management"
|
||||
_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management"
|
||||
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
_CLAUDE_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
class _RoutesTo(CustomRoutingStrategyBase):
|
||||
def __init__(self, router: Router, api_key: str) -> None:
|
||||
self._router: Final = router
|
||||
self._api_key: Final = api_key
|
||||
|
||||
async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]:
|
||||
return next(
|
||||
deployment
|
||||
for deployment in self._router.model_list
|
||||
if deployment["litellm_params"]["api_key"] == self._api_key
|
||||
)
|
||||
|
||||
|
||||
def _mixed_drop_group(routed_key: str) -> Router:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": _KEEPS_CONTEXT_MANAGEMENT},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-haiku-5-5",
|
||||
"api_key": _DROPS_CONTEXT_MANAGEMENT,
|
||||
"additional_drop_params": ["context_management"],
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
router.set_custom_routing_strategy(_RoutesTo(router, routed_key))
|
||||
return router
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("routed_key", (_KEEPS_CONTEXT_MANAGEMENT, _DROPS_CONTEXT_MANAGEMENT))
|
||||
async def test_anthropic_count_tokens_in_a_mixed_drop_group_sends_context_management_when_the_routed_inference_does(
|
||||
monkeypatch: pytest.MonkeyPatch, routed_key: str
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key))
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
messages: Final = [{"role": "user", "content": "Weather in SF?"}]
|
||||
|
||||
with respx.mock:
|
||||
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||
)
|
||||
count_route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 7})
|
||||
)
|
||||
await _mixed_drop_group(routed_key).aanthropic_messages(
|
||||
model="claude-agent", messages=messages, max_tokens=16, context_management=_CONTEXT_EDITS
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(
|
||||
_json_request(
|
||||
{"model": "claude-agent", "messages": messages, "context_management": _CONTEXT_EDITS},
|
||||
"python-httpx/0.28.1",
|
||||
),
|
||||
UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
inference: Final = inference_route.calls.last.request
|
||||
count: Final = count_route.calls.last.request
|
||||
assert inference.headers["x-api-key"] == count.headers["x-api-key"] == routed_key
|
||||
assert ("context_management" in json.loads(count.content)) == (
|
||||
"context_management" in json.loads(inference.content)
|
||||
)
|
||||
assert ("context_management" in json.loads(count.content)) == (routed_key == _KEEPS_CONTEXT_MANAGEMENT)
|
||||
|
||||
|
||||
_AZURE_AI_CLAUDE: Final[dict[str, str]] = {
|
||||
"model": "azure_ai/claude-haiku-5-5",
|
||||
"api_base": "https://example-resource.services.ai.azure.com/anthropic",
|
||||
"api_key": "azure-test-key",
|
||||
}
|
||||
_ANTHROPIC_CLAUDE: Final[dict[str, str]] = {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-ant-test"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "general_settings", "forwarded_model_groups", "caller_beta_sent"),
|
||||
(
|
||||
(_ANTHROPIC_CLAUDE, {}, None, True),
|
||||
(_AZURE_AI_CLAUDE, {}, None, False),
|
||||
(_AZURE_AI_CLAUDE, {"forward_client_headers_to_llm_api": True}, None, True),
|
||||
(_AZURE_AI_CLAUDE, {}, ["claude-agent"], True),
|
||||
),
|
||||
ids=(
|
||||
"anthropic-api-header-provider",
|
||||
"other-provider",
|
||||
"other-provider-forwarding-client-headers",
|
||||
"other-provider-in-a-forwarding-model-group",
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_sends_the_caller_beta_only_when_inference_does(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
litellm_params: dict[str, str],
|
||||
general_settings: dict[str, object],
|
||||
forwarded_model_groups: list[str] | None,
|
||||
caller_beta_sent: bool,
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
from litellm.types.router import ModelGroupSettings
|
||||
|
||||
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": litellm_params}])
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(proxy_server, "general_settings", general_settings)
|
||||
monkeypatch.setattr(
|
||||
litellm, "model_group_settings", ModelGroupSettings(forward_client_headers_to_llm_api=forwarded_model_groups)
|
||||
)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"model": "claude-agent",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
}
|
||||
caller_headers: Final = {"anthropic-beta": _CALLER_BETA}
|
||||
proxy_data: Final = await add_litellm_data_to_request(
|
||||
data=dict(body),
|
||||
request=_json_request(body, "python-httpx/0.28.1", caller_headers),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"),
|
||||
proxy_config=proxy_server.ProxyConfig(),
|
||||
general_settings=general_settings,
|
||||
)
|
||||
|
||||
with respx.mock:
|
||||
inference_route: Final = respx.post(url__regex=r".*/v1/messages$").mock(
|
||||
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||
)
|
||||
count_route: Final = respx.post(url__regex=r".*/v1/messages/count_tokens$").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 7})
|
||||
)
|
||||
await router.aanthropic_messages(
|
||||
**body,
|
||||
max_tokens=4096,
|
||||
**{name: proxy_data[name] for name in ("provider_specific_header", "headers") if name in proxy_data},
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(
|
||||
_json_request(body, "python-httpx/0.28.1", caller_headers), UserAPIKeyAuth(api_key="sk-1234")
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
inference_betas: Final = inference_route.calls.last.request.headers.get("anthropic-beta", "").split(",")
|
||||
count_betas: Final = count_route.calls.last.request.headers.get("anthropic-beta", "").split(",")
|
||||
assert (_CALLER_BETA in inference_betas) == (_CALLER_BETA in count_betas) == caller_beta_sent
|
||||
|
||||
|
||||
async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
await litellm.anthropic_messages(
|
||||
model=f"anthropic/{model}",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
max_tokens=4096,
|
||||
api_key="sk-ant-test",
|
||||
**options,
|
||||
)
|
||||
return json.loads(route.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_drop_params", "user_agent"),
|
||||
(
|
||||
({"drop_params": True}, "python-httpx/0.28.1"),
|
||||
({}, "claude-cli/2.1.0 (external, cli)"),
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send(
|
||||
monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
|
||||
dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True})
|
||||
kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking})
|
||||
assert dropped["thinking"] != kept["thinking"]
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-opus-4-7",
|
||||
{"thinking": thinking, **request_drop_params},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
user_agent,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
counted: Final = json.loads(count.content)
|
||||
assert {name: counted.get(name) for name in ("thinking", "output_config")} == {
|
||||
name: dropped.get(name) for name in ("thinking", "output_config")
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_drop_params", "request_params"),
|
||||
(
|
||||
(True, {"drop_params": "maybe"}),
|
||||
(True, {"drop_params": False}),
|
||||
(True, {"drop_params": None}),
|
||||
(True, {}),
|
||||
(None, {"drop_params": " true "}),
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_deployments_like_inference(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_params: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
|
||||
deployment_params: Final[Mapping[str, JsonValue]] = (
|
||||
{} if deployment_drop_params is None else {"drop_params": deployment_drop_params}
|
||||
)
|
||||
with respx.mock:
|
||||
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||
)
|
||||
await _claude_router("anthropic/claude-opus-5-5", deployment_params).aanthropic_messages(
|
||||
model="claude-agent",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
max_tokens=4096,
|
||||
thinking=thinking,
|
||||
**request_params,
|
||||
)
|
||||
sent: Final = json.loads(inference_route.calls.last.request.content)
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-opus-5-5",
|
||||
{"thinking": thinking, **request_params},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
deployment_params=deployment_params,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert json.loads(count.content).get("thinking") == sent.get("thinking")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
output_format: Final[dict[str, JsonValue]] = {
|
||||
"type": "json_schema",
|
||||
"schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
}
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-haiku-5-5",
|
||||
{"output_format": output_format},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
counted: Final = json.loads(count.content)
|
||||
assert "output_format" not in counted
|
||||
assert counted["output_config"]["format"] == output_format
|
||||
|
||||
|
||||
class _ScriptedCounter(BaseTokenCounter):
|
||||
def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None:
|
||||
self._replies = replies
|
||||
|
|
@ -1299,6 +1974,8 @@ class _ScriptedCounter(BaseTokenCounter):
|
|||
messages_format: TokenCountMessagesFormat = "openai",
|
||||
output_format: JsonValue = None,
|
||||
context_management: object = None,
|
||||
request_options: object = None,
|
||||
anthropic_beta: object = None,
|
||||
) -> TokenCountResponse | None:
|
||||
self.calls.append((model_to_use, messages, tools, system))
|
||||
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
|
||||
|
|
@ -3037,7 +3714,7 @@ class _PartnerCountRequest(BaseModel):
|
|||
|
||||
|
||||
async def _vertex_partner_word_count(
|
||||
model: str, request_data: JsonValue, litellm_params: JsonValue
|
||||
model: str, request_data: JsonValue, litellm_params: JsonValue, betas: object = ()
|
||||
) -> dict[str, JsonValue]:
|
||||
body: Final = _PartnerCountRequest.model_validate(request_data)
|
||||
return {"input_tokens": sum(len(message.content.split()) for message in body.messages)}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue