mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): forward context_management, thinking, tool_choice and output_config to the Claude count APIs
The Anthropic-format count on /v1/messages/count_tokens dropped these fields on anthropic, azure_ai, vertex_ai and bedrock (runtime and Mantle), so the count disagreed with what inference sends. The count now runs the deployment's own /v1/messages transform and sends the same anthropic-beta values as inference, from one shared context_management_betas helper
This commit is contained in:
parent
b7da9a2755
commit
f012c60033
23 changed files with 730 additions and 101 deletions
|
|
@ -12,7 +12,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
|||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -45,6 +45,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -347,6 +348,20 @@ def _beta_header_values(side: str | Sequence[str] | None) -> tuple[str, ...]:
|
|||
return tuple(entry for entry in side if isinstance(entry, str))
|
||||
|
||||
|
||||
def context_management_betas(context_management: JsonValue) -> frozenset[str]:
|
||||
"""The anthropic-beta values a ``context_management`` body needs: the compact beta for a
|
||||
``compact_20260112`` edit and the context-management beta for every other edit."""
|
||||
edits: Final = context_management.get("edits") if isinstance(context_management, dict) else None
|
||||
if not isinstance(edits, list):
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
|
||||
if isinstance(edit, dict) and edit.get("type") == "compact_20260112"
|
||||
else ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
|
||||
for edit in edits
|
||||
)
|
||||
|
||||
|
||||
def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta: str | Sequence[str] | None) -> str:
|
||||
"""Merge anthropic-beta header values, deduplicated and sorted.
|
||||
|
||||
|
|
@ -1306,8 +1321,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Anthropic CountTokens API handler.
|
|||
Uses httpx for HTTP requests instead of the Anthropic SDK.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
betas: Sequence[str] = (),
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
Handle a CountTokens request using httpx.
|
||||
|
|
@ -79,7 +80,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
verbose_logger.debug("Making request to: %s", endpoint_url)
|
||||
|
||||
# Get required headers
|
||||
headers: Final = self.get_count_tokens_headers(auth_header)
|
||||
headers: Final = self.get_count_tokens_headers(auth_header, betas)
|
||||
|
||||
# Use LiteLLM's async httpx client
|
||||
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
|
|||
AnthropicCountFields,
|
||||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
claude_count_options,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
|
@ -49,6 +50,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
|
|
@ -93,6 +95,12 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
case None:
|
||||
pass
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.ANTHROPIC, request_options, context_management, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "anthropic_api")
|
||||
|
||||
deployment = deployment or {}
|
||||
litellm_params: Final = deployment.get("litellm_params", {})
|
||||
api_base: Final = litellm_params.get("api_base")
|
||||
|
|
@ -130,7 +138,8 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
api_base=api_base,
|
||||
tools=tools,
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
optional_params={**count_options, **claude_options.options},
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -29,12 +29,12 @@ from litellm.litellm_core_utils.url_utils import join_api_base
|
|||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
|
||||
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
|
||||
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
|
||||
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
_PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"})
|
||||
|
||||
|
||||
|
|
@ -113,17 +113,24 @@ class ChatCountTranslationError:
|
|||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore"))
|
||||
class _AnthropicChatRequest(TypedDict):
|
||||
messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]]
|
||||
system: NotRequired[ReadOnly[JsonValue]]
|
||||
tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]]
|
||||
class _CountedOptions(TypedDict):
|
||||
thinking: NotRequired[ReadOnly[JsonValue]]
|
||||
tool_choice: NotRequired[ReadOnly[JsonValue]]
|
||||
output_config: NotRequired[ReadOnly[JsonValue]]
|
||||
output_format: NotRequired[ReadOnly[JsonValue]]
|
||||
context_management: NotRequired[ReadOnly[JsonValue]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore"))
|
||||
class _AnthropicChatRequest(_CountedOptions):
|
||||
messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]]
|
||||
system: NotRequired[ReadOnly[JsonValue]]
|
||||
tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]]
|
||||
|
||||
|
||||
_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest)
|
||||
_COUNTED_OPTIONS: Final = TypeAdapter(_CountedOptions)
|
||||
_INFERENCE_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_ChatParams: TypeAlias = dict[str, object]
|
||||
_ChatMessages: TypeAlias = list[AllMessageValues]
|
||||
|
||||
|
|
@ -240,7 +247,7 @@ def _system_messages(system: object) -> tuple[Mapping[str, object], ...]:
|
|||
return (message,) if isinstance(system, (str, list)) and system else ()
|
||||
|
||||
|
||||
def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]:
|
||||
def _count_options(request: _CountedOptions) -> Mapping[str, JsonValue]:
|
||||
"""Structured output reaches chat inference as the beta ``output_format``, which count_tokens
|
||||
refuses; its GA ``output_config.format`` form counts the same schema the same way."""
|
||||
output_config: Final = request.get("output_config")
|
||||
|
|
@ -256,6 +263,7 @@ def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]:
|
|||
("thinking", request.get("thinking")),
|
||||
("tool_choice", request.get("tool_choice")),
|
||||
("output_config", counted_config or None),
|
||||
("context_management", request.get("context_management")),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
|
|
@ -325,6 +333,77 @@ def anthropic_fields_for_chat_tool_use(
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ClaudeCountOptions:
|
||||
options: Mapping[str, JsonValue]
|
||||
betas: tuple[str, ...]
|
||||
|
||||
|
||||
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=())
|
||||
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
|
||||
|
||||
|
||||
def claude_count_options(
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
request_options: Mapping[str, JsonValue],
|
||||
context_management: CountContextManagement | None,
|
||||
deployment: Mapping[str, object] | None,
|
||||
) -> ClaudeCountOptions | ChatCountTranslationError:
|
||||
"""The thinking, tool_choice, output_config and context_management a Claude count sends, with the
|
||||
anthropic-beta values they need, after the deployment's own /v1/messages transform rewrites them.
|
||||
|
||||
The transform runs without messages and with a max_tokens above any thinking budget, so it never
|
||||
caps a budget the caller's real max_tokens would allow. An error it raises is the error inference
|
||||
answers for the same body.
|
||||
"""
|
||||
sent: Final = {
|
||||
**{name: value for name, value in request_options.items() if value is not None},
|
||||
**({} if context_management is None else {"context_management": context_management.spec}),
|
||||
}
|
||||
if not sent:
|
||||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, context_management_betas
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
|
||||
if messages_config is None:
|
||||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
resolved_deployment: Final = deployment or {}
|
||||
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
try:
|
||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||
model=model,
|
||||
messages=[],
|
||||
anthropic_messages_optional_request_params={
|
||||
**copy.deepcopy(sent),
|
||||
"max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET,
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams.model_validate(
|
||||
{
|
||||
"drop_params": deployment_params.get("drop_params"),
|
||||
"model_info": resolved_deployment.get("model_info"),
|
||||
}
|
||||
),
|
||||
headers={},
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
options: Final = _count_options(_COUNTED_OPTIONS.validate_python(request))
|
||||
betas: Final = sorted(context_management_betas(options.get("context_management")))
|
||||
return ClaudeCountOptions(
|
||||
options=options,
|
||||
betas=tuple(
|
||||
filter_and_transform_beta_headers(betas, provider.value)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else betas
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def anthropic_count_fields(
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
|
|
@ -402,7 +481,7 @@ class AnthropicCountTokensConfig:
|
|||
)
|
||||
)
|
||||
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str], betas: Sequence[str] = ()) -> dict[str, str]:
|
||||
"""The count-tokens headers around a resolved Anthropic auth header
|
||||
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
|
||||
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
|
||||
|
|
@ -412,7 +491,7 @@ class AnthropicCountTokensConfig:
|
|||
"anthropic-version": "2023-06-01",
|
||||
**auth_header,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
auth_header.get("anthropic-beta"), (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *betas)
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from ...common_utils import (
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
merge_anthropic_beta_headers,
|
||||
optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta,
|
||||
|
|
@ -792,28 +793,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", ())
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
|
||||
|
||||
# Check for structured outputs. Anthropic's newer request shape nests
|
||||
# the schema under output_config.format; the older top-level
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Azure AI Anthropic CountTokens API handler.
|
|||
Uses httpx for HTTP requests with Azure authentication.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -13,7 +13,7 @@ from pydantic import JsonValue
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, merge_anthropic_beta_headers
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
|
|
@ -38,6 +38,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: object = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
betas: Sequence[str] = (),
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Handle a CountTokens request using httpx with Azure authentication.
|
||||
|
|
@ -101,7 +102,10 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
|
||||
response: Final = await async_client.post(
|
||||
endpoint_url,
|
||||
headers=headers,
|
||||
headers={
|
||||
**headers,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(headers.get("anthropic-beta"), betas),
|
||||
},
|
||||
json=request_body,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
|
|||
AnthropicCountFields,
|
||||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
claude_count_options,
|
||||
)
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
|
||||
AzureAIAnthropicCountTokensHandler,
|
||||
|
|
@ -63,6 +64,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
|
|
@ -107,6 +109,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
case None:
|
||||
pass
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.AZURE_AI, request_options, context_management, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
||||
|
||||
deployment = deployment or {}
|
||||
litellm_params: Final = deployment.get("litellm_params", {})
|
||||
|
||||
|
|
@ -134,7 +142,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
litellm_params=litellm_params,
|
||||
tools=tools,
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
optional_params={**count_options, **claude_options.options},
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ class BaseTokenCounter(ABC):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,13 @@ from typing_extensions import assert_never
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
NO_CLAUDE_COUNT_OPTIONS,
|
||||
ChatCountTranslationError,
|
||||
ClaudeCountOptions,
|
||||
anthropic_count_fields,
|
||||
claude_count_options,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
|
||||
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
|
||||
|
|
@ -74,6 +80,13 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
|
|||
)
|
||||
|
||||
|
||||
def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]:
|
||||
return {
|
||||
**claude_options.options,
|
||||
**({"anthropic_beta": list(claude_options.betas)} if claude_options.betas else {}),
|
||||
}
|
||||
|
||||
|
||||
def _request_data(
|
||||
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, system: object
|
||||
) -> Mapping[str, object]:
|
||||
|
|
@ -143,6 +156,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
if not messages:
|
||||
return None
|
||||
|
|
@ -167,11 +181,27 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
)
|
||||
if isinstance(translated, ChatCountTranslationError):
|
||||
return translated.response(request_model, model_to_use, primary_tokenizer)
|
||||
runtime_request: Final = (
|
||||
_request_data(model_to_use, messages, tools, system)
|
||||
if translated is None
|
||||
else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
|
||||
claude_options: Final = (
|
||||
claude_count_options(
|
||||
counted_model,
|
||||
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
|
||||
request_options,
|
||||
context_management,
|
||||
deployment,
|
||||
)
|
||||
if is_mantle_claude_model(resolved_model)
|
||||
else NO_CLAUDE_COUNT_OPTIONS
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, primary_tokenizer)
|
||||
runtime_request: Final = {
|
||||
**(
|
||||
_request_data(model_to_use, messages, tools, system)
|
||||
if translated is None
|
||||
else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
|
||||
),
|
||||
**_invoke_fields(claude_options),
|
||||
}
|
||||
mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
|
||||
|
||||
primary: Final = (
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class MantleCountTokensRequest(TypedDict):
|
|||
_COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
|
||||
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
|
||||
_BETAS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class BedrockMantleCountTokensHandler(BaseAWSLLM):
|
||||
|
|
@ -70,10 +71,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
|
|||
)
|
||||
verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
|
||||
api_key: Final = litellm_params.get("api_key")
|
||||
betas: Final = _BETAS.validate_python(request_data.get("anthropic_beta", ()))
|
||||
signed_headers, signed_body = await run_aws_signing(
|
||||
self._sign_request,
|
||||
service_name="bedrock",
|
||||
headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": MANTLE_ANTHROPIC_VERSION,
|
||||
**({"anthropic-beta": ",".join(betas)} if betas else {}),
|
||||
},
|
||||
optional_params=litellm_params,
|
||||
request_data=body,
|
||||
api_base=endpoint_url,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,16 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
|
|||
# max_tokens field; CountTokens only counts input, so it has no effect
|
||||
# on any generation.
|
||||
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS: Final = 1024
|
||||
_INVOKE_MODEL_ONLY_FIELDS: Final = frozenset(
|
||||
("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta")
|
||||
)
|
||||
|
||||
|
||||
def _invoke_model_max_tokens(thinking: JsonValue) -> int:
|
||||
budget: Final = thinking.get("budget_tokens") if isinstance(thinking, dict) else None
|
||||
if not isinstance(budget, int):
|
||||
return DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
|
||||
return max(DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, budget + 1)
|
||||
|
||||
|
||||
def _json_dict(value: JsonValue) -> dict[str, JsonValue]:
|
||||
|
|
@ -87,6 +97,8 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
'converse' or 'invokeModel'
|
||||
"""
|
||||
messages: Final = request_data.get("messages")
|
||||
if not _INVOKE_MODEL_ONLY_FIELDS.isdisjoint(request_data):
|
||||
return "invokeModel"
|
||||
if isinstance(messages, list):
|
||||
# Anthropic content blocks carry a "type" key ({"type": "text", ...});
|
||||
# Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
|
||||
|
|
@ -200,7 +212,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
# Bedrock validates the body against the model's InvokeModel schema;
|
||||
# Anthropic Messages bodies require these fields.
|
||||
body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
|
||||
body_data.setdefault("max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS)
|
||||
body_data.setdefault("max_tokens", _invoke_model_max_tokens(body_data.get("thinking")))
|
||||
|
||||
# The CountTokens API expects invokeModel.body as a base64-encoded blob
|
||||
encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode()
|
||||
|
|
|
|||
|
|
@ -534,6 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
Count tokens using OpenAI's Responses API /input_tokens endpoint.
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from litellm.types.llms.vertex_ai import (
|
|||
PartType,
|
||||
Schema,
|
||||
)
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.utils import supports_response_schema, supports_system_messages
|
||||
|
||||
GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
|
||||
|
|
@ -1391,8 +1391,6 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
self,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> bool:
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
return custom_llm_provider == LlmProviders.VERTEX_AI.value
|
||||
|
||||
async def _count_translated_claude_request(
|
||||
|
|
@ -1454,6 +1452,7 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
|
@ -1508,6 +1507,17 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
if translated_count is not None:
|
||||
return translated_count
|
||||
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
ChatCountTranslationError,
|
||||
claude_count_options,
|
||||
)
|
||||
|
||||
claude_options: Final = claude_count_options(
|
||||
model_to_use, LlmProviders.VERTEX_AI, request_options, context_management, deployment
|
||||
)
|
||||
if isinstance(claude_options, ChatCountTranslationError):
|
||||
return claude_options.response(request_model, model_to_use, "vertex_ai_partner_models")
|
||||
|
||||
# Use partner models token counter
|
||||
partner_models_handler: Final = VertexAIPartnerModels()
|
||||
|
||||
|
|
@ -1538,7 +1548,8 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
vertex_credentials=vertex_credentials,
|
||||
system=system,
|
||||
tools=tools,
|
||||
count_options=count_options,
|
||||
count_options={**count_options, **claude_options.options},
|
||||
betas=claude_options.betas,
|
||||
)
|
||||
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
|
||||
status_code: Final = e.response.status_code
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
messages_carry_output_config,
|
||||
|
|
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
if existing_beta:
|
||||
beta_values.update(b.strip() for b in existing_beta.split(","))
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", [])
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management"))) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is the config contract's bare dict
|
||||
|
||||
# Check for web search tool
|
||||
for tool in tools:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Unlike Gemini models which use Google's token counting API, partner models use
|
|||
their respective publisher-specific count-tokens endpoints.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
|
|
@ -90,6 +91,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
model: str,
|
||||
request_data: dict[str, Any],
|
||||
litellm_params: dict[str, Any],
|
||||
betas: Sequence[str] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Handle token counting request for a Vertex AI partner model.
|
||||
|
|
@ -98,6 +100,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
model: The model name
|
||||
request_data: Request payload (Anthropic Messages API format)
|
||||
litellm_params: LiteLLM parameters containing credentials, project, location
|
||||
betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header
|
||||
|
||||
Returns:
|
||||
Dict containing token count information
|
||||
|
|
@ -157,7 +160,10 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
)
|
||||
|
||||
# Prepare headers
|
||||
headers: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
headers: Final = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
**({"anthropic-beta": ",".join(betas)} if betas else {}),
|
||||
}
|
||||
|
||||
# Get async HTTP client
|
||||
from litellm import LlmProviders
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# What is this?
|
||||
## API Handler for calling Vertex AI Partner Models
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -267,6 +267,7 @@ class VertexAIPartnerModels(VertexBase):
|
|||
system: object | None = None,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
count_options: Mapping[str, object] = MappingProxyType({}),
|
||||
betas: Sequence[str] = (),
|
||||
):
|
||||
"""
|
||||
Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
|
||||
|
|
@ -321,6 +322,7 @@ class VertexAIPartnerModels(VertexBase):
|
|||
model=model,
|
||||
request_data=request_data,
|
||||
litellm_params=_litellm_params,
|
||||
betas=betas,
|
||||
)
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -336,12 +336,14 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
|
|||
|
||||
|
||||
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
|
||||
_COUNTED_REQUEST_OPTIONS: Final = ("thinking", "tool_choice", "output_config", "output_format")
|
||||
|
||||
|
||||
class _CountTokensBodyOptions(TypedDict):
|
||||
output_format: ReadOnly[JsonValue]
|
||||
context_management: ReadOnly[JsonValue]
|
||||
request_litellm_params: ReadOnly[Mapping[str, JsonValue]]
|
||||
request_options: ReadOnly[Mapping[str, JsonValue]]
|
||||
|
||||
|
||||
def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions:
|
||||
|
|
@ -352,6 +354,9 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
|
|||
"request_litellm_params": MappingProxyType(
|
||||
{"drop_params": True, **sent_params} if auto_drop_params else sent_params
|
||||
),
|
||||
"request_options": MappingProxyType(
|
||||
{key: data[key] for key in _COUNTED_REQUEST_OPTIONS if data.get(key) is not None}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -299,7 +299,7 @@ def generate_feedback_box():
|
|||
import contextlib
|
||||
from collections import defaultdict
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from functools import lru_cache, partial
|
||||
from functools import lru_cache, partial, reduce
|
||||
|
||||
import litellm
|
||||
import litellm._redis
|
||||
|
|
@ -1374,6 +1374,8 @@ _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody)
|
|||
_LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody)
|
||||
_ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig)
|
||||
_LITELLM_PARAMS_COPY: Final = TypeAdapter(dict[str, object])
|
||||
_COUNT_REQUEST_OPTIONS: Final = TypeAdapter(Mapping[str, JsonValue])
|
||||
_DROPPED_PARAM_PATHS: Final = TypeAdapter(tuple[object, ...])
|
||||
_WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings)
|
||||
_UNTYPED_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
|
@ -14810,6 +14812,7 @@ async def _try_provider_token_count(
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
|
||||
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
|
||||
|
|
@ -14837,6 +14840,7 @@ async def _try_provider_token_count(
|
|||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
chat_params=chat_params,
|
||||
request_options=request_options,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
return ProviderCountFailure(
|
||||
|
|
@ -15239,6 +15243,20 @@ def _count_context_management(
|
|||
)
|
||||
|
||||
|
||||
def _count_request_options(
|
||||
request_options: Mapping[str, JsonValue],
|
||||
deployment: Mapping[str, object] | None,
|
||||
sent_params: Mapping[str, object],
|
||||
) -> Mapping[str, JsonValue]:
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
|
||||
deployment_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
|
||||
dropped: Final = {**deployment_params, **sent_params}.get("additional_drop_params")
|
||||
dropped_values: Final = _DROPPED_PARAM_PATHS.validate_python(dropped) if isinstance(dropped, list) else ()
|
||||
dropped_paths: Final = tuple(path for path in dropped_values if isinstance(path, str))
|
||||
return _COUNT_REQUEST_OPTIONS.validate_python(reduce(delete_nested_value, dropped_paths, dict(request_options)))
|
||||
|
||||
|
||||
def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
previous_response_id: Final = None if responses_request is None else responses_request.get("previous_response_id")
|
||||
return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {}
|
||||
|
|
@ -15316,6 +15334,7 @@ async def count_request_tokens(
|
|||
responses_request: Mapping[str, object] | None = None,
|
||||
litellm_call_id: str | None = None,
|
||||
client_oauth_token: str | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse:
|
||||
call_id: Final = resolve_litellm_call_id(litellm_call_id)
|
||||
if request.prompt is None and request.messages is None and request.contents is None:
|
||||
|
|
@ -15335,6 +15354,7 @@ async def count_request_tokens(
|
|||
request_litellm_params=request_litellm_params,
|
||||
responses_request=responses_request,
|
||||
client_oauth_token=client_oauth_token,
|
||||
request_options=request_options,
|
||||
)
|
||||
return await logged_token_count(
|
||||
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
|
||||
|
|
@ -15377,6 +15397,7 @@ async def _count_request_tokens(
|
|||
request_litellm_params: Mapping[str, JsonValue],
|
||||
responses_request: Mapping[str, object] | None,
|
||||
client_oauth_token: str | None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -15484,6 +15505,7 @@ async def _count_request_tokens(
|
|||
output_format=output_format,
|
||||
context_management=_count_context_management(context_management, request.model, request_litellm_params),
|
||||
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
|
||||
request_options=_count_request_options(request_options, count_deployment, sent_params),
|
||||
),
|
||||
model_to_use=model_to_use,
|
||||
request_model=request.model,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import respx
|
|||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
|
||||
|
|
@ -26,7 +27,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
|
|||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -475,6 +477,43 @@ _COUNTERS: Final = pytest.mark.parametrize(
|
|||
)
|
||||
|
||||
|
||||
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
async def _counted_requests(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
messages: list[dict[str, JsonValue]],
|
||||
tools: list[dict[str, JsonValue]] | None,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
context_management: CountContextManagement | None = None,
|
||||
model: str = "claude-opus-5-5",
|
||||
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def count(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(200, json={"input_tokens": 31})
|
||||
|
||||
with respx.mock:
|
||||
respx.post(count_endpoint).mock(side_effect=count)
|
||||
result: Final = await counter_type().count_tokens(
|
||||
model_to_use=model,
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
request_model=model,
|
||||
tools=tools,
|
||||
messages_format=messages_format,
|
||||
chat_params=chat_params,
|
||||
request_options=request_options,
|
||||
context_management=context_management,
|
||||
)
|
||||
return result, tuple(sent.get() for _ in range(sent.qsize()))
|
||||
|
||||
|
||||
async def _counted_body(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
|
|
@ -483,25 +522,10 @@ async def _counted_body(
|
|||
messages_format: TokenCountMessagesFormat,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]:
|
||||
sent: Final[SimpleQueue[bytes]] = SimpleQueue()
|
||||
|
||||
def count(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request.content)
|
||||
return httpx.Response(200, json={"input_tokens": 31})
|
||||
|
||||
with respx.mock:
|
||||
respx.post(count_endpoint).mock(side_effect=count)
|
||||
result: Final = await counter_type().count_tokens(
|
||||
model_to_use="claude-opus-5-5",
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
request_model="claude-opus-5-5",
|
||||
tools=tools,
|
||||
messages_format=messages_format,
|
||||
chat_params=chat_params,
|
||||
)
|
||||
return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize()))
|
||||
result, requests = await _counted_requests(
|
||||
counter_type, count_endpoint, messages, tools, messages_format, chat_params
|
||||
)
|
||||
return result, tuple(_JSON_BODY.validate_json(request.content) for request in requests)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -842,3 +866,134 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat
|
|||
inference["tool_choice"],
|
||||
inference["tools"],
|
||||
)
|
||||
|
||||
|
||||
_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
_INFERENCE_PROVIDERS: Final = MappingProxyType(
|
||||
{AnthropicTokenCounter: "anthropic", AzureAIAnthropicTokenCounter: "azure_ai"}
|
||||
)
|
||||
_CONTEXT_BETAS: Final = frozenset(
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
)
|
||||
)
|
||||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}
|
||||
_ENABLED_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000}
|
||||
|
||||
|
||||
async def _messages_inference_request(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
inference_endpoint: str,
|
||||
request_options: Mapping[str, JsonValue],
|
||||
) -> httpx.Request:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def reply(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(200, json=_MESSAGE_REPLY)
|
||||
|
||||
with respx.mock:
|
||||
respx.post(inference_endpoint).mock(side_effect=reply)
|
||||
await litellm.anthropic_messages(
|
||||
model=f"{_INFERENCE_PROVIDERS[counter_type]}/claude-haiku-5-5",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
max_tokens=4096,
|
||||
api_key="sk-ant-api03-test-key",
|
||||
api_base="https://gateway.example",
|
||||
**request_options,
|
||||
)
|
||||
return sent.get()
|
||||
|
||||
|
||||
def _betas(request: httpx.Request) -> frozenset[str]:
|
||||
return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "context_management"),
|
||||
(
|
||||
({"thinking": _ENABLED_THINKING}, None),
|
||||
({"tool_choice": {"type": "any"}}, None),
|
||||
({"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, None),
|
||||
(
|
||||
{"thinking": _ENABLED_THINKING},
|
||||
{"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]},
|
||||
),
|
||||
({}, {"edits": [{"type": "compact_20260112"}]}),
|
||||
),
|
||||
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
|
||||
)
|
||||
async def test_counter_sends_the_options_and_context_edits_messages_inference_sends(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
request_options: dict[str, JsonValue],
|
||||
context_management: dict[str, JsonValue] | None,
|
||||
) -> None:
|
||||
"""A /v1/messages count that drops thinking, tool_choice, output_config or context_management
|
||||
counts a different prompt than inference bills, so it carries them the way inference sends them."""
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type,
|
||||
count_endpoint.removesuffix("/count_tokens"),
|
||||
{**request_options, **({} if context_management is None else {"context_management": context_management})},
|
||||
)
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
request_options=request_options,
|
||||
context_management=(
|
||||
None if context_management is None else CountContextManagement(spec=context_management, drop_invalid_edits=False)
|
||||
),
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
inference_body: Final = _JSON_BODY.validate_json(inference.content)
|
||||
count_body: Final = _JSON_BODY.validate_json(count.content)
|
||||
inference_output_format: Final = inference_body.get("output_format")
|
||||
inference_output_config: Final = inference_body.get("output_config")
|
||||
assert "output_format" not in count_body
|
||||
assert count_body.get("output_config") == (
|
||||
inference_output_config
|
||||
if inference_output_format is None
|
||||
else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format}
|
||||
)
|
||||
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
|
||||
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
|
||||
}
|
||||
assert _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) & _CONTEXT_BETAS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_refuses_options_messages_inference_refuses_without_calling_anthropic(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
result, requests = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
None,
|
||||
"anthropic",
|
||||
request_options={"output_config": {"effort": "fastest"}},
|
||||
model="claude-haiku-4-5",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
assert requests == ()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter
|
|||
import litellm
|
||||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.utils import CountContextManagement
|
||||
|
||||
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
|
|
@ -338,6 +339,105 @@ async def test_claude_count_sends_the_tool_search_tool_the_deployments_inference
|
|||
assert counted_tools == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
|
||||
|
||||
|
||||
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
}
|
||||
_THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
"tool_choice": {"type": "auto"},
|
||||
}
|
||||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}}
|
||||
|
||||
|
||||
async def _messages_inference_and_count(deployment_model: str, model_to_use: str) -> tuple[_Bedrock, _Bedrock]:
|
||||
credentials: Final = DEPLOYMENT["litellm_params"]
|
||||
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=MESSAGES,
|
||||
tools=[_WEATHER_TOOL],
|
||||
max_tokens=4096,
|
||||
context_management=_CONTEXT_EDITS,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
|
||||
**_THINKING_AND_TOOL_CHOICE,
|
||||
**credentials,
|
||||
)
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use=model_to_use,
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {**credentials, "model": deployment_model}},
|
||||
request_model="claude",
|
||||
tools=[_WEATHER_TOOL],
|
||||
request_options=_THINKING_AND_TOOL_CHOICE,
|
||||
context_management=CountContextManagement(spec=_CONTEXT_EDITS, drop_invalid_edits=False),
|
||||
)
|
||||
assert result is not None
|
||||
assert result.error is False
|
||||
return inference, bedrock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "model_to_use"),
|
||||
(
|
||||
("bedrock/invoke/global.anthropic.claude-haiku-5-5", "global.anthropic.claude-haiku-5-5"),
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"),
|
||||
),
|
||||
)
|
||||
async def test_runtime_count_sends_the_options_and_context_edits_invoke_inference_sends(
|
||||
deployment_model: str, model_to_use: str
|
||||
) -> None:
|
||||
"""Bedrock invoke inference drops edits InvokeModel rejects and sends its betas in the body, so a count
|
||||
that skips either counts a prompt inference never sends, or gets the 400 inference avoids."""
|
||||
inference, bedrock = await _messages_inference_and_count(deployment_model, model_to_use)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
counted: Final = _invoke_body(bedrock.requests[0])
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta")
|
||||
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
|
||||
thinking: Final = counted["thinking"]
|
||||
assert isinstance(thinking, dict)
|
||||
budget_tokens: Final = thinking.get("budget_tokens", 0)
|
||||
assert isinstance(budget_tokens, int) and isinstance(counted["max_tokens"], int)
|
||||
assert counted["max_tokens"] > budget_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mantle_count_sends_the_options_context_edits_and_beta_mantle_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
"bedrock_mantle/us.anthropic.claude-opus-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5"
|
||||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (MANTLE_HOST,)
|
||||
counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content)
|
||||
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields}
|
||||
assert "anthropic_beta" not in counted
|
||||
assert bedrock.requests[0].headers["anthropic-beta"] == inference.requests[0].headers["anthropic-beta"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None:
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment=DEPLOYMENT,
|
||||
request_model="claude-sonnet-4-5",
|
||||
request_options={"output_config": {"effort": "fastest"}},
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
assert bedrock.posted_hosts() == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_chat_tool_use_counted_on_mantle_metadata_route_is_translated_too() -> None:
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import copy
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from enum import Enum
|
||||
from queue import SimpleQueue
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
|
@ -1840,7 +1842,9 @@ _PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = {
|
|||
|
||||
|
||||
async def _partner_counted_bodies(
|
||||
model: str, chat_params: dict[str, JsonValue] | None = None
|
||||
model: str,
|
||||
chat_params: dict[str, JsonValue] | None = None,
|
||||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
|
||||
|
|
@ -1875,6 +1879,7 @@ async def _partner_counted_bodies(
|
|||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=[_PARTNER_RESPONSES_TOOL],
|
||||
chat_params=chat_params or {},
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
|
|
@ -1954,6 +1959,101 @@ async def test_vertex_claude_count_sends_the_structured_output_vertex_inference_
|
|||
)
|
||||
|
||||
|
||||
_PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
||||
context_edits: Final[dict[str, JsonValue]] = {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
}
|
||||
options: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
"tool_choice": {"type": "auto"},
|
||||
}
|
||||
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
|
||||
tools: Final[list[dict[str, JsonValue]]] = [{"name": "get_weather", "input_schema": {"type": "object"}}]
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def vertex(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(
|
||||
200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY
|
||||
)
|
||||
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex))
|
||||
with (
|
||||
patch.object(handler, "get_async_httpx_client", return_value=client),
|
||||
patch.object(
|
||||
VertexBase, "_ensure_access_token_async", new=AsyncMock(return_value=("fake-token", "test-project"))
|
||||
),
|
||||
):
|
||||
await litellm.anthropic_messages(
|
||||
model="vertex_ai/claude-haiku-5-5",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
max_tokens=4096,
|
||||
context_management=context_edits,
|
||||
vertex_project="test-project",
|
||||
vertex_location="us-east5",
|
||||
client=client,
|
||||
**options,
|
||||
)
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=tools,
|
||||
request_options=options,
|
||||
context_management=CountContextManagement(spec=context_edits, drop_invalid_edits=False),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 37
|
||||
inference, count = tuple(sent.get() for _ in range(sent.qsize()))
|
||||
assert count.url.path.endswith("count-tokens:rawPredict")
|
||||
counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content)
|
||||
inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
assert {name: counted.get(name) for name in fields} == {name: inferred.get(name) for name in fields}
|
||||
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None:
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use="claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
request_options={"output_config": {"effort": "fastest"}},
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_llama_count_leaves_out_the_claude_only_options() -> None:
|
||||
model: Final = "meta/llama-4-maverick-17b-128e-instruct-maas"
|
||||
|
||||
assert await _partner_counted_bodies(
|
||||
model, request_options={"thinking": {"type": "enabled", "budget_tokens": 2000}}
|
||||
) == await _partner_counted_bodies(model)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_llama_count_sends_chat_tool_use_as_sent() -> None:
|
||||
assert await _partner_counted_bodies("meta/llama-4-maverick-17b-128e-instruct-maas") == (
|
||||
|
|
|
|||
|
|
@ -367,6 +367,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "claude-3-sonnet-20240229"
|
||||
|
|
@ -449,6 +450,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "gpt-4"
|
||||
|
|
@ -1269,6 +1271,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"detail":{"message":"Input is too long for requested model."}}',
|
||||
|
|
@ -1328,6 +1331,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"Message":"Bearer Token has expired"}',
|
||||
|
|
@ -1387,6 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
request_options=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
|
|
@ -1607,6 +1612,103 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t
|
|||
assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}]
|
||||
|
||||
|
||||
|
||||
_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "adaptive"},
|
||||
"tool_choice": {"type": "auto"},
|
||||
"output_config": {
|
||||
"effort": "low",
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
}
|
||||
|
||||
|
||||
def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
|
||||
output_config: Final = body.get("output_config")
|
||||
return frozenset(name for name in _CLAUDE_COUNT_OPTIONS if name in body) | frozenset(
|
||||
f"output_config.{key}" for key in (output_config if isinstance(output_config, dict) else {})
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("additional_drop_params", "expected_paths"),
|
||||
(
|
||||
(
|
||||
None,
|
||||
{
|
||||
"thinking",
|
||||
"tool_choice",
|
||||
"output_config",
|
||||
"output_config.effort",
|
||||
"output_config.format",
|
||||
"context_management",
|
||||
},
|
||||
),
|
||||
(
|
||||
["tool_choice", "output_config.format"],
|
||||
{"thinking", "output_config", "output_config.effort", "context_management"},
|
||||
),
|
||||
(
|
||||
["context_management"],
|
||||
{"thinking", "tool_choice", "output_config", "output_config.effort", "output_config.format"},
|
||||
),
|
||||
),
|
||||
ids=("nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management"),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
|
||||
monkeypatch: pytest.MonkeyPatch, additional_drop_params: list[str] | None, expected_paths: set[str]
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-haiku-5-5",
|
||||
"api_key": "sk-ant-test",
|
||||
"additional_drop_params": additional_drop_params,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
request: Final = _json_request(
|
||||
{
|
||||
"model": "claude-agent",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
|
||||
**_CLAUDE_COUNT_OPTIONS,
|
||||
},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 7})
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
|
||||
assert response["input_tokens"] == 7
|
||||
sent_request: Final = route.calls.last.request
|
||||
assert _sent_option_paths(json.loads(sent_request.content)) == expected_paths
|
||||
assert ("context-management-2025-06-27" in sent_request.headers["anthropic-beta"]) == (
|
||||
"context_management" in expected_paths
|
||||
)
|
||||
|
||||
|
||||
class _ScriptedCounter(BaseTokenCounter):
|
||||
def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None:
|
||||
self._replies = replies
|
||||
|
|
@ -1631,6 +1733,7 @@ class _ScriptedCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: object = None,
|
||||
chat_params: object = None,
|
||||
request_options: object = None,
|
||||
) -> TokenCountResponse | None:
|
||||
self.calls.append((model_to_use, messages, tools, system))
|
||||
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
|
||||
|
|
@ -3470,7 +3573,7 @@ class _PartnerCountRequest(BaseModel):
|
|||
|
||||
|
||||
async def _vertex_partner_word_count(
|
||||
model: str, request_data: JsonValue, litellm_params: JsonValue
|
||||
model: str, request_data: JsonValue, litellm_params: JsonValue, betas: object = ()
|
||||
) -> dict[str, JsonValue]:
|
||||
body: Final = _PartnerCountRequest.model_validate(request_data)
|
||||
return {"input_tokens": sum(len(message.content.split()) for message in body.messages)}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue