fix(proxy): count thinking, tool_choice, and effort the way inference sends them, and count DeepSeek on its own count route

/v1/messages/count_tokens dropped the body's thinking, tool_choice, and output_config, so a Claude count with
tool_choice any came out lower than the bill. The route now passes those options to the provider counter, which
runs them through the deployment's own /v1/messages transform first, so the count sees what inference sends.
A body inference would reject now answers the same 400 before any provider call.

DeepSeek had no provider counter and fell back to a local estimate that ignores thinking. It now counts on
DeepSeek's Anthropic-compatible count_tokens route with the same base, key, tool sanitizer, and reasoning
mapping inference uses, and falls back to the estimate only when that route is missing.
This commit is contained in:
mateo-berri 2026-10-29 08:43:29 -07:00
parent dff474e450
commit 493cf79031
22 changed files with 1165 additions and 147 deletions

View file

@ -2,7 +2,7 @@
Anthropic Token Counter implementation using the CountTokens API.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final
@ -11,11 +11,13 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
anthropic_count_fields,
messages_count_options,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
@ -64,12 +66,11 @@ class AnthropicTokenCounter(BaseTokenCounter):
Returns:
TokenCountResponse with token count, or None if counting fails
"""
from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo
if not messages:
return None
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
match translated := await anthropic_count_fields(
model_to_use, messages, system, tools, messages_format, AnthropicConfig(), chat_params
@ -77,24 +78,51 @@ class AnthropicTokenCounter(BaseTokenCounter):
case ChatCountTranslationError():
return translated.response(request_model, model_to_use, "anthropic_api")
case AnthropicCountFields():
return await self.count_tokens(
return await self._count(
model_to_use=model_to_use,
messages=[dict(message) for message in translated.messages],
contents=contents,
messages=translated.messages,
deployment=deployment,
request_model=request_model,
tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None,
tools=translated.tools,
system=translated.system,
messages_format="anthropic",
output_format=output_format,
context_management=context_management,
count_options=translated.options,
)
case None:
pass
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
match options := await asyncify(messages_count_options)(
AnthropicMessagesConfig(),
model_to_use,
messages,
count_options,
(deployment or {}).get("litellm_params", {}),
):
case ChatCountTranslationError():
return options.response(request_model, model_to_use, "anthropic_api")
case _:
return await self._count(
model_to_use=model_to_use,
messages=messages,
deployment=deployment,
request_model=request_model,
tools=tools,
system=system,
count_options=options,
)
async def _count(
self,
model_to_use: str,
messages: Sequence[Mapping[str, JsonValue]],
deployment: Mapping[str, Any] | None,
request_model: str,
tools: Sequence[Mapping[str, JsonValue]] | None,
system: JsonValue,
count_options: Mapping[str, JsonValue],
) -> TokenCountResponse | None:
from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo
litellm_params: Final = (deployment or {}).get("litellm_params", {})
api_base: Final = litellm_params.get("api_base")
try:
@ -125,10 +153,10 @@ class AnthropicTokenCounter(BaseTokenCounter):
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,
messages=messages,
messages=[dict(message) for message in messages],
auth_header=auth_header,
api_base=api_base,
tools=tools,
tools=None if tools is None else [dict(tool) for tool in tools],
system=system,
optional_params=count_options,
)

View file

@ -7,9 +7,11 @@ This module handles the transformation of requests to Anthropic's CountTokens AP
import copy
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import reduce
from itertools import chain
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Final,
Protocol,
TypeAlias,
@ -19,8 +21,9 @@ from typing import (
from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_blocks,
split_leading_system_run,
@ -29,9 +32,16 @@ from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
if TYPE_CHECKING:
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
_COUNTED_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]])
_TRANSFORMED_REQUEST: Final = TypeAdapter(dict[str, object])
_DROPPED_PATHS: Final = TypeAdapter(tuple[str, ...])
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
@ -325,6 +335,64 @@ def anthropic_fields_for_chat_tool_use(
)
def _max_output_tokens(model: str, custom_llm_provider: str | None) -> int:
from litellm.utils import get_model_info
try:
max_output_tokens: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider).get(
"max_output_tokens"
)
except Exception: # noqa: BLE001 # get_model_info raises a plain Exception for a model the cost map lacks
return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
return (
max_output_tokens
if isinstance(max_output_tokens, int) and max_output_tokens > 0
else DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
)
def without_dropped_paths(options: Mapping[str, JsonValue], additional_drop_params: object) -> Mapping[str, JsonValue]:
return _COUNT_REQUEST.validate_python(
reduce(delete_nested_value, _DROPPED_PATHS.validate_python(additional_drop_params or ()), {**options})
)
def messages_count_options(
translator: "AnthropicMessagesConfig",
model: str,
messages: Sequence[Mapping[str, JsonValue]],
options: Mapping[str, JsonValue],
litellm_params: Mapping[str, object],
) -> Mapping[str, JsonValue] | ChatCountTranslationError:
"""The thinking, tool_choice, and output_config the deployment's /v1/messages transform sends upstream
for these Anthropic Messages body options. The transform needs max_tokens, which a count request lacks,
so it gets the model's max output tokens: a thinking budget's size never changes a count, only whether
thinking stays on."""
if not options:
return MappingProxyType({})
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError
try:
request: Final = _TRANSFORMED_REQUEST.validate_python(
translator.transform_anthropic_messages_request(
model=model,
messages=_COUNTED_MESSAGES.validate_python(messages),
anthropic_messages_optional_request_params={
**without_dropped_paths(options, litellm_params.get("additional_drop_params")),
"max_tokens": _max_output_tokens(model, translator.custom_llm_provider),
},
litellm_params=GenericLiteLLMParams.model_validate({"drop_params": litellm_params.get("drop_params")}),
headers={},
)
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
return _COUNT_REQUEST.validate_python(
{name: request[name] for name in COUNT_TOKEN_OPTION_NAMES if request.get(name) is not None}
)
async def anthropic_count_fields(
model: str,
messages: Sequence[Mapping[str, object]],

View file

@ -3,7 +3,7 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API.
"""
import os
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final
@ -11,10 +11,12 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
anthropic_count_fields,
messages_count_options,
)
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
@ -78,11 +80,10 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
Returns:
TokenCountResponse with token count, or None if counting fails
"""
from litellm.llms.anthropic.common_utils import AnthropicError
if not messages:
return None
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
match translated := await anthropic_count_fields(
@ -91,24 +92,51 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
case ChatCountTranslationError():
return translated.response(request_model, model_to_use, "azure_ai_anthropic_api")
case AnthropicCountFields():
return await self.count_tokens(
return await self._count(
model_to_use=model_to_use,
messages=[dict(message) for message in translated.messages],
contents=contents,
messages=translated.messages,
deployment=deployment,
request_model=request_model,
tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None,
tools=translated.tools,
system=translated.system,
messages_format="anthropic",
output_format=output_format,
context_management=context_management,
count_options=translated.options,
)
case None:
pass
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
match options := await asyncify(messages_count_options)(
AzureAnthropicMessagesConfig(),
model_to_use,
messages,
count_options,
(deployment or {}).get("litellm_params", {}),
):
case ChatCountTranslationError():
return options.response(request_model, model_to_use, "azure_ai_anthropic_api")
case _:
return await self._count(
model_to_use=model_to_use,
messages=messages,
deployment=deployment,
request_model=request_model,
tools=tools,
system=system,
count_options=options,
)
async def _count(
self,
model_to_use: str,
messages: Sequence[Mapping[str, JsonValue]],
deployment: Mapping[str, Any] | None,
request_model: str,
tools: Sequence[Mapping[str, JsonValue]] | None,
system: object | None,
count_options: Mapping[str, JsonValue],
) -> TokenCountResponse | None:
from litellm.llms.anthropic.common_utils import AnthropicError
litellm_params: Final = (deployment or {}).get("litellm_params", {})
# Get Azure AI API key from deployment config or environment
api_key = litellm_params.get("api_key")
@ -128,11 +156,11 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
try:
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,
messages=messages,
messages=[dict(message) for message in messages],
api_key=api_key,
api_base=api_base,
litellm_params=litellm_params,
tools=tools,
tools=None if tools is None else [dict(tool) for tool in tools],
system=system,
optional_params=count_options,
)

View file

@ -42,6 +42,7 @@ class BaseTokenCounter(ABC):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
pass

View file

@ -143,6 +143,7 @@ class BedrockTokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
if not messages:
return None

View file

@ -3,9 +3,11 @@ Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completi
"""
from collections.abc import Coroutine, Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal, cast, overload
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
extract_search_results_text,
@ -20,6 +22,35 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
def _thinking_type(thinking: object, reasoning_effort: object) -> str | None:
requested: Final = as_str_mapping(thinking)
requested_type: Final = None if requested is None else requested.get("type")
if isinstance(requested_type, str) and requested_type in ("enabled", "disabled"):
return requested_type
if reasoning_effort is None:
return None
return "disabled" if reasoning_effort == "none" else "enabled"
def deepseek_reasoning_params(thinking: object, reasoning_effort: object) -> Mapping[str, object]:
"""DeepSeek takes `{"type": "enabled"}` or `{"type": "disabled"}` with no budget_tokens. A thinking
param wins. Otherwise `reasoning_effort="none"` turns thinking off and any other effort turns it on,
and a supported effort is sent only while thinking is on.
Reference: https://api-docs.deepseek.com/guides/thinking_mode
"""
thinking_type: Final = _thinking_type(thinking, reasoning_effort)
if thinking_type is None:
return MappingProxyType({})
if (
thinking_type == "enabled"
and isinstance(reasoning_effort, str)
and reasoning_effort in SUPPORTED_REASONING_EFFORTS
):
return MappingProxyType({"thinking": {"type": thinking_type}, "reasoning_effort": reasoning_effort})
return MappingProxyType({"thinking": {"type": thinking_type}})
class DeepSeekChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
"""
@ -36,40 +67,10 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
model: str,
drop_params: bool,
) -> dict[str, object]:
"""
Map OpenAI params to DeepSeek params.
Handles `thinking` and `reasoning_effort` parameters for DeepSeek reasoner models.
DeepSeek supports `{"type": "enabled"}` and `{"type": "disabled"}` - no budget_tokens
like Anthropic. `reasoning_effort="none"` is the OpenAI-style way to ask for thinking
off, so it maps to `{"type": "disabled"}`; any other effort keeps thinking on.
Reference: https://api-docs.deepseek.com/guides/thinking_mode
"""
# Let parent handle standard params first
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
# Pop thinking/reasoning_effort from optional_params first (parent may have added them)
# Then re-add only if valid for DeepSeek
thinking_value: Final = optional_params.pop("thinking", None)
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
# Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):
optional_params["thinking"] = {"type": thinking_value["type"]}
# Otherwise fall back to reasoning_effort: "none" disables, anything else enables
elif reasoning_effort is not None:
optional_params["thinking"] = {"type": "disabled" if reasoning_effort == "none" else "enabled"}
if (
isinstance(reasoning_effort, str)
and reasoning_effort in SUPPORTED_REASONING_EFFORTS
and optional_params["thinking"] == {"type": "enabled"}
):
return {**optional_params, "reasoning_effort": reasoning_effort}
return optional_params
return {**optional_params, **deepseek_reasoning_params(thinking_value, reasoning_effort)}
def _fill_reasoning_content(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
"""

View file

@ -0,0 +1,46 @@
from collections.abc import Mapping, Sequence
from typing import Final
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig, deepseek_anthropic_base
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
class DeepSeekModelInfo(BaseLLMModelInfo):
def get_models(
self, api_key: str | None = None, api_base: str | None = None
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
from litellm.utils import get_valid_models
return get_valid_models(custom_llm_provider=LlmProviders.DEEPSEEK.value)
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return DeepSeekAnthropicMessagesConfig.get_api_key(api_key)
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
return deepseek_anthropic_base(api_base)
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: matches validate_environment's dict contract shared by every provider override
resolved_api_key: Final = self.get_api_key(api_key)
return {**headers, **({} if resolved_api_key is None else {"x-api-key": resolved_api_key})}
@staticmethod
def get_base_model(model: str) -> str | None:
return model.removeprefix(f"{LlmProviders.DEEPSEEK.value}/")
def get_token_counter(self) -> BaseTokenCounter | None:
from litellm.llms.deepseek.count_tokens.token_counter import DeepSeekTokenCounter
return DeepSeekTokenCounter()

View file

@ -0,0 +1,211 @@
"""
DeepSeek token counting through the count_tokens route of its Anthropic-compatible API.
"""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from pydantic import JsonValue, TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
anthropic_count_fields,
messages_count_options,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.deepseek.chat.transformation import deepseek_reasoning_params
from litellm.llms.deepseek.common_utils import DeepSeekModelInfo
from litellm.llms.deepseek.messages.transformation import (
DeepSeekAnthropicMessagesConfig,
sanitize_tools_for_deepseek,
)
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
DEEPSEEK_TOKENIZER_TYPE: Final = "deepseek_api"
_CHAT_REASONING_PARAMS: Final = ("thinking", "reasoning_effort")
_COUNT_ROUTE_MISSING: Final = frozenset({404, 405})
_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]])
_OPTIONS: Final = TypeAdapter(dict[str, JsonValue])
deepseek_count_tokens_handler: Final = AnthropicCountTokensHandler()
def _chat_reasoning_options(chat_params: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
mapped: Final = deepseek_reasoning_params(chat_params.get("thinking"), chat_params.get("reasoning_effort"))
effort: Final = mapped.get("reasoning_effort")
return _OPTIONS.validate_python(
{
**({} if mapped.get("thinking") is None else {"thinking": mapped["thinking"]}),
**({} if effort is None else {"output_config": {"effort": effort}}),
}
)
def _with_chat_reasoning(
options: Mapping[str, JsonValue], reasoning: Mapping[str, JsonValue]
) -> Mapping[str, JsonValue]:
output_config: Final = options.get("output_config")
reasoning_output_config: Final = reasoning.get("output_config")
return _OPTIONS.validate_python(
{
**options,
**reasoning,
**(
{"output_config": {**output_config, **reasoning_output_config}}
if isinstance(output_config, dict) and isinstance(reasoning_output_config, dict)
else {}
),
}
)
def _litellm_params(deployment: Mapping[str, object] | None) -> Mapping[str, object]:
params: Final = None if deployment is None else as_str_mapping(deployment.get("litellm_params"))
return MappingProxyType({}) if params is None else params
def _optional_str(value: object) -> str | None:
return value if isinstance(value, str) else None
class DeepSeekTokenCounter(BaseTokenCounter):
"""Counts with the request DeepSeek bills: an Anthropic Messages body sent to the Anthropic-compatible
count_tokens route, with thinking and effort mapped the way the deployment's inference maps them."""
def __init__(self, handler: AnthropicCountTokensHandler = deepseek_count_tokens_handler) -> None:
self._handler: Final = handler
def should_use_token_counting_api(self, custom_llm_provider: str | None = None) -> bool:
return custom_llm_provider == LlmProviders.DEEPSEEK.value
async def count_tokens(
self,
model_to_use: str,
messages: Sequence[Mapping[str, JsonValue]] | None,
contents: Sequence[Mapping[str, object]] | None,
deployment: Mapping[str, object] | None = None,
request_model: str = "",
tools: Sequence[Mapping[str, JsonValue]] | None = None,
system: JsonValue = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
if not messages:
return None
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
litellm_params: Final = _litellm_params(deployment)
match translated := await anthropic_count_fields(
model_to_use,
messages,
system,
tools,
messages_format,
AnthropicConfig(),
{name: value for name, value in chat_params.items() if name not in _CHAT_REASONING_PARAMS},
):
case ChatCountTranslationError():
return translated.response(request_model, model_to_use, DEEPSEEK_TOKENIZER_TYPE)
case AnthropicCountFields():
return await self._count(
model_to_use=model_to_use,
messages=translated.messages,
litellm_params=litellm_params,
request_model=request_model,
tools=translated.tools,
system=translated.system,
count_options=_with_chat_reasoning(translated.options, _chat_reasoning_options(chat_params)),
)
case None:
pass
if messages_format == "openai":
return await self._count(
model_to_use=model_to_use,
messages=messages,
litellm_params=litellm_params,
request_model=request_model,
tools=tools,
system=system,
count_options=_chat_reasoning_options(chat_params),
)
match options := await asyncify(messages_count_options)(
DeepSeekAnthropicMessagesConfig(), model_to_use, messages, count_options, litellm_params
):
case ChatCountTranslationError():
return options.response(request_model, model_to_use, DEEPSEEK_TOKENIZER_TYPE)
case _:
return await self._count(
model_to_use=model_to_use,
messages=messages,
litellm_params=litellm_params,
request_model=request_model,
tools=tools,
system=system,
count_options=options,
)
async def _count(
self,
model_to_use: str,
messages: Sequence[Mapping[str, JsonValue]],
litellm_params: Mapping[str, object],
request_model: str,
tools: Sequence[Mapping[str, JsonValue]] | None,
system: JsonValue,
count_options: Mapping[str, JsonValue],
) -> TokenCountResponse | None:
model_info: Final = DeepSeekModelInfo()
try:
result: Final = await self._handler.handle_count_tokens_request(
model=model_to_use,
messages=[dict(message) for message in messages],
auth_header=model_info.validate_environment(
headers={},
model=model_to_use,
messages=(),
optional_params={},
litellm_params=litellm_params,
api_key=_optional_str(litellm_params.get("api_key")),
),
api_base=model_info.get_api_base(_optional_str(litellm_params.get("api_base"))),
tools=None if tools is None else _TOOLS.validate_python(sanitize_tools_for_deepseek(tools)),
system=system,
optional_params=count_options,
)
except AnthropicError as e:
if e.status_code in _COUNT_ROUTE_MISSING:
verbose_logger.debug(
"DeepSeek count_tokens route answered %s, so the count is estimated", e.status_code
)
return None
verbose_logger.warning("DeepSeek CountTokens error: status=%s, message=%s", e.status_code, e.message)
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type=DEEPSEEK_TOKENIZER_TYPE,
error=True,
error_message=e.message,
status_code=e.status_code,
)
input_tokens: Final = result.get("input_tokens")
return TokenCountResponse(
total_tokens=input_tokens if isinstance(input_tokens, int) else 0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type=DEEPSEEK_TOKENIZER_TYPE,
original_response=result,
)

View file

@ -16,6 +16,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
_TOOL_LIST: Final = TypeAdapter(list[object])
_REQUEST_FIELDS: Final = TypeAdapter(dict[str, object])
_TOOL_FIELDS: Final = TypeAdapter(dict[object, object])
@ -34,6 +35,20 @@ def _sanitized_tool(tool: object) -> object:
return tool if stripped is None else stripped
def sanitize_tools_for_deepseek(tools: Sequence[object]) -> Sequence[object]:
return [_sanitized_tool(tool) for tool in tools]
def deepseek_anthropic_base(api_base: str | None) -> str:
configured: Final = DeepSeekAnthropicMessagesConfig.get_api_base(api_base=api_base).rstrip("/")
if configured.endswith("/v1/messages") and "/anthropic/" in configured:
return configured.removesuffix("/v1/messages")
stripped: Final = configured.removesuffix("/v1/messages").removesuffix("/v1").removesuffix("/beta")
if stripped.endswith("/anthropic") or "/anthropic/" in stripped:
return stripped
return join_api_base(stripped, "anthropic")
class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
DeepSeek exposes an Anthropic-compatible Messages API at
@ -102,25 +117,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
litellm_params: dict,
stream: bool | None = None,
) -> str:
base_url = self.get_api_base(api_base=api_base).rstrip("/")
if base_url.endswith("/v1/messages") and "/anthropic/" in base_url:
return base_url
base_url = base_url.removesuffix("/v1/messages")
base_url = base_url.removesuffix("/v1")
base_url = base_url.removesuffix("/beta")
if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url:
base_url = join_api_base(base_url, "anthropic")
return join_api_base(base_url, "v1/messages")
@staticmethod
def _sanitize_tools_for_deepseek(tools: object) -> object:
if not isinstance(tools, list):
return tools
return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)]
return join_api_base(deepseek_anthropic_base(api_base), "v1/messages")
def transform_anthropic_messages_request(
self,
@ -130,13 +127,20 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
anthropic_messages_request: Final = super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
anthropic_messages_request: Final = _REQUEST_FIELDS.validate_python(
super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
)
if "tools" in anthropic_messages_request:
anthropic_messages_request["tools"] = self._sanitize_tools_for_deepseek(anthropic_messages_request["tools"])
return anthropic_messages_request
tools: Final = anthropic_messages_request.get("tools")
if not isinstance(tools, list):
return anthropic_messages_request
sanitized: Final = {
**anthropic_messages_request,
"tools": sanitize_tools_for_deepseek(_TOOL_LIST.validate_python(tools)),
}
return sanitized

View file

@ -534,6 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
import copy

View file

@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse | None:
"""
Count tokens using OpenAI's Responses API /input_tokens endpoint.

View file

@ -7,7 +7,7 @@ from enum import Enum
from functools import lru_cache
from itertools import accumulate, chain, repeat, takewhile
from types import MappingProxyType
from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast, get_type_hints
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -33,6 +33,9 @@ from litellm.types.llms.vertex_ai import (
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
from litellm.utils import supports_response_schema, supports_system_messages
if TYPE_CHECKING:
from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountFields
GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
{"fps": "fps", "start_offset": "startOffset", "end_offset": "endOffset"}
)
@ -1395,21 +1398,17 @@ class VertexAITokenCounter(BaseTokenCounter):
return custom_llm_provider == LlmProviders.VERTEX_AI.value
async def _count_translated_claude_request(
async def _translated_claude_request(
self,
model_to_use: str,
messages: Sequence[Mapping[str, object]] | None,
deployment: Mapping[str, object],
request_model: str,
tools: Sequence[Mapping[str, object]] | None,
system: object | None,
messages_format: TokenCountMessagesFormat,
output_format: JsonValue,
context_management: CountContextManagement | None,
chat_params: Mapping[str, JsonValue],
) -> TokenCountResponse | None:
) -> "TokenCountResponse | AnthropicCountFields | None":
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
anthropic_count_fields,
)
@ -1419,27 +1418,12 @@ class VertexAITokenCounter(BaseTokenCounter):
return None
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
match translated := await anthropic_count_fields(
translated: Final = await anthropic_count_fields(
model_to_use, messages or [], system, tools, messages_format, VertexAIAnthropicConfig(), chat_params
):
case ChatCountTranslationError():
return translated.response(request_model, model_to_use, "vertex_ai_partner_models")
case AnthropicCountFields():
return await self.count_tokens(
model_to_use=model_to_use,
messages=[dict(message) for message in translated.messages],
contents=None,
deployment=dict(deployment),
request_model=request_model,
tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None,
system=translated.system,
messages_format="anthropic",
output_format=output_format,
context_management=context_management,
count_options=translated.options,
)
case None:
return None
)
if isinstance(translated, ChatCountTranslationError):
return translated.response(request_model, model_to_use, "vertex_ai_partner_models")
return translated
async def count_tokens(
self,
@ -1493,20 +1477,17 @@ class VertexAITokenCounter(BaseTokenCounter):
# Check if this is a partner model (Claude, Mistral, etc.)
if VertexAIPartnerModels.is_vertex_partner_model(model_to_use):
translated_count: Final = await self._count_translated_claude_request(
translated: Final = await self._translated_claude_request(
model_to_use=model_to_use,
messages=messages,
deployment=deployment,
request_model=request_model,
tools=tools,
system=system,
messages_format=messages_format,
output_format=output_format,
context_management=context_management,
chat_params=chat_params,
)
if translated_count is not None:
return translated_count
if isinstance(translated, TokenCountResponse):
return translated
# Use partner models token counter
partner_models_handler: Final = VertexAIPartnerModels()
@ -1531,14 +1512,22 @@ class VertexAITokenCounter(BaseTokenCounter):
try:
result = await partner_models_handler.count_tokens(
model=model_to_use,
messages=messages or [],
messages=(
(messages or []) if translated is None else [dict(message) for message in translated.messages]
),
litellm_params=partner_litellm_params,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_credentials=vertex_credentials,
system=system,
tools=tools,
count_options=count_options,
system=system if translated is None else translated.system,
tools=(
tools
if translated is None
else None
if translated.tools is None
else [dict(tool) for tool in translated.tools]
),
count_options=MappingProxyType({}) if translated is None else translated.options,
)
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
status_code: Final = e.response.status_code

View file

@ -1,6 +1,6 @@
# What is this?
## API Handler for calling Vertex AI Partner Models
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Final
@ -259,13 +259,13 @@ class VertexAIPartnerModels(VertexBase):
async def count_tokens(
self,
model: str,
messages: list,
litellm_params: dict,
vertex_project=None,
vertex_location=None,
vertex_credentials=None,
messages: Sequence[Mapping[str, object]],
litellm_params: Mapping[str, object],
vertex_project: object = None,
vertex_location: object = None,
vertex_credentials: object = None,
system: object | None = None,
tools: list[dict[str, object]] | None = None,
tools: Sequence[Mapping[str, object]] | None = None,
count_options: Mapping[str, object] = MappingProxyType({}),
):
"""
@ -307,13 +307,18 @@ class VertexAIPartnerModels(VertexBase):
}
# Prepare litellm_params with credentials
_litellm_params: Final = litellm_params.copy()
if vertex_project:
_litellm_params["vertex_project"] = vertex_project
if vertex_location:
_litellm_params["vertex_location"] = vertex_location
if vertex_credentials:
_litellm_params["vertex_credentials"] = vertex_credentials
_litellm_params: Final = {
**litellm_params,
**{
key: value
for key, value in (
("vertex_project", vertex_project),
("vertex_location", vertex_location),
("vertex_credentials", vertex_credentials),
)
if value
},
}
# Call the token counter
token_counter: Final = VertexAIPartnerModelsTokenCounter()

View file

@ -19,6 +19,7 @@ from litellm.anthropic_interface.exceptions import (
)
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, without_dropped_paths
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
)
@ -342,6 +343,7 @@ class _CountTokensBodyOptions(TypedDict):
output_format: ReadOnly[JsonValue]
context_management: ReadOnly[JsonValue]
request_litellm_params: ReadOnly[Mapping[str, JsonValue]]
count_options: ReadOnly[Mapping[str, JsonValue]]
def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions:
@ -352,6 +354,12 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
"request_litellm_params": MappingProxyType(
{"drop_params": True, **sent_params} if auto_drop_params else sent_params
),
"count_options": MappingProxyType(
without_dropped_paths(
{name: data[name] for name in COUNT_TOKEN_OPTION_NAMES if name in data},
data.get("additional_drop_params"),
)
),
}

View file

@ -14814,6 +14814,7 @@ async def _try_provider_token_count(
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
@ -14841,6 +14842,7 @@ async def _try_provider_token_count(
output_format=output_format,
context_management=context_management,
chat_params=chat_params,
count_options=count_options,
)
except httpx.HTTPStatusError as e:
return ProviderCountFailure(
@ -15320,6 +15322,7 @@ async def count_request_tokens(
responses_request: Mapping[str, object] | None = None,
litellm_call_id: str | None = None,
client_oauth_token: str | None = None,
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
) -> TokenCountResponse:
call_id: Final = resolve_litellm_call_id(litellm_call_id)
if request.prompt is None and request.messages is None and request.contents is None:
@ -15339,6 +15342,7 @@ async def count_request_tokens(
request_litellm_params=request_litellm_params,
responses_request=responses_request,
client_oauth_token=client_oauth_token,
count_options=count_options,
)
return await logged_token_count(
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
@ -15381,6 +15385,7 @@ async def _count_request_tokens(
request_litellm_params: Mapping[str, JsonValue],
responses_request: Mapping[str, object] | None,
client_oauth_token: str | None,
count_options: Mapping[str, JsonValue],
) -> TokenCountResponse:
prompt: Final = request.prompt
messages: Final = request.messages
@ -15488,6 +15493,7 @@ async def _count_request_tokens(
output_format=output_format,
context_management=_count_context_management(context_management, request.model, request_litellm_params),
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
count_options=count_options,
),
model_to_use=model_to_use,
request_model=request.model,

View file

@ -10142,6 +10142,10 @@ class ProviderConfigManager:
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
return AzureFoundryModelInfo(model=model)
elif LlmProviders.DEEPSEEK == provider:
from litellm.llms.deepseek.common_utils import DeepSeekModelInfo
return DeepSeekModelInfo()
return None
@staticmethod

View file

@ -16,9 +16,12 @@ import litellm
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.count_tokens.transformation import (
COUNT_TOKEN_OPTION_NAMES,
AnthropicCountTokensConfig,
)
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
@ -26,6 +29,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -482,6 +487,8 @@ async def _counted_body(
tools: list[dict[str, JsonValue]] | None,
messages_format: TokenCountMessagesFormat,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
model: str = "claude-opus-5-5",
) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]:
sent: Final[SimpleQueue[bytes]] = SimpleQueue()
@ -492,14 +499,15 @@ async def _counted_body(
with respx.mock:
respx.post(count_endpoint).mock(side_effect=count)
result: Final = await counter_type().count_tokens(
model_to_use="claude-opus-5-5",
model_to_use=model,
messages=messages,
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
request_model="claude-opus-5-5",
request_model=model,
tools=tools,
messages_format=messages_format,
chat_params=chat_params,
count_options=count_options,
)
return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize()))
@ -842,3 +850,112 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat
inference["tool_choice"],
inference["tools"],
)
_MESSAGES_CONFIGS: Final = MappingProxyType(
{AnthropicTokenCounter: AnthropicMessagesConfig, AzureAIAnthropicTokenCounter: AzureAnthropicMessagesConfig}
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
@pytest.mark.parametrize(
("model", "options"),
(
("claude-sonnet-4-5", {"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}),
("claude-haiku-5-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}}),
("claude-opus-5-5", {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}),
("claude-unlisted-model", {"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}),
),
)
async def test_counter_sends_the_thinking_tool_choice_and_effort_messages_inference_sends(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
model: str,
options: Mapping[str, JsonValue],
) -> None:
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
inference: Final = _MESSAGES_CONFIGS[counter_type]().transform_anthropic_messages_request(
model=model,
messages=deepcopy(messages),
anthropic_messages_optional_request_params={**deepcopy(dict(options)), "max_tokens": 4096},
litellm_params=GenericLiteLLMParams(),
headers={},
)
result, bodies = await _counted_body(
counter_type, count_endpoint, messages, None, "anthropic", count_options=options, model=model
)
(body,) = bodies
assert result is not None
assert result.total_tokens == 31
assert {name: body.get(name) for name in COUNT_TOKEN_OPTION_NAMES} == {
name: inference.get(name) for name in COUNT_TOKEN_OPTION_NAMES
}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_leaves_out_an_option_the_deployment_drops(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
sent: Final[SimpleQueue[bytes]] = SimpleQueue()
def count(request: httpx.Request) -> httpx.Response:
sent.put(request.content)
return httpx.Response(200, json={"input_tokens": 31})
with respx.mock:
respx.post(count_endpoint).mock(side_effect=count)
await counter_type().count_tokens(
model_to_use="claude-sonnet-4-5",
messages=[{"role": "user", "content": "Weather in SF?"}],
contents=None,
deployment={
"litellm_params": {
"api_key": "sk-ant-api03-test-key",
"api_base": "https://gateway.example",
"additional_drop_params": ["tool_choice"],
}
},
request_model="claude-sonnet-4-5",
messages_format="anthropic",
count_options={"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}},
)
body: Final = TypeAdapter(dict[str, JsonValue]).validate_json(sent.get())
assert ("tool_choice" in body, body["thinking"]) == (False, {"type": "enabled", "budget_tokens": 2048})
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_rejects_a_thinking_budget_messages_inference_rejects_without_calling_anthropic(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
options: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": "lots"}}
with pytest.raises(BadRequestError) as rejected:
_MESSAGES_CONFIGS[counter_type]().transform_anthropic_messages_request(
model="claude-opus-5-5",
messages=[{"role": "user", "content": "Weather in SF?"}],
anthropic_messages_optional_request_params={**deepcopy(options), "max_tokens": 4096},
litellm_params=GenericLiteLLMParams(),
headers={},
)
result, bodies = await _counted_body(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
None,
"anthropic",
count_options=options,
)
assert bodies == ()
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
assert rejected.value.message in (result.error_message or "")

View file

@ -0,0 +1,213 @@
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
import pytest
import respx
from pydantic import JsonValue
import litellm
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.deepseek.count_tokens.token_counter import DeepSeekTokenCounter
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
_DEFAULT_COUNT_ROUTE: Final = "https://api.deepseek.com/anthropic/v1/messages/count_tokens"
_WEATHER_IN_SF: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
_GET_WEATHER: Final[dict[str, JsonValue]] = {
"name": "get_weather",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
}
@pytest.fixture(autouse=True)
def _deepseek_env_unset(monkeypatch: pytest.MonkeyPatch) -> None:
for name in ("DEEPSEEK_API_BASE", "DEEPSEEK_ANTHROPIC_API_BASE", "DEEPSEEK_API_KEY"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(litellm, "api_key", None)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
async def _count(
reply: httpx.Response,
count_route: str = _DEFAULT_COUNT_ROUTE,
litellm_params: Mapping[str, JsonValue] = MappingProxyType({"api_key": "sk-deepseek-test"}),
messages_format: TokenCountMessagesFormat = "anthropic",
tools: list[dict[str, JsonValue]] | None = None,
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
messages: list[dict[str, JsonValue]] = _WEATHER_IN_SF,
) -> tuple[TokenCountResponse | None, respx.Route]:
with respx.mock:
route: Final = respx.post(count_route).mock(return_value=reply)
result: Final = await DeepSeekTokenCounter().count_tokens(
model_to_use="deepseek-v4-pro",
messages=messages,
contents=None,
deployment={"litellm_params": dict(litellm_params)},
request_model="deepseek-pro",
tools=tools,
messages_format=messages_format,
chat_params=chat_params,
count_options=count_options,
)
return result, route
@pytest.mark.asyncio
@pytest.mark.parametrize(
"api_base",
(
None,
"https://api.deepseek.com",
"https://api.deepseek.com/beta",
"https://api.deepseek.com/anthropic/v1/messages",
"https://gateway.example/deepseek",
),
)
async def test_count_posts_under_the_base_messages_inference_posts_to(api_base: str | None) -> None:
messages_url: Final = DeepSeekAnthropicMessagesConfig().get_complete_url(
api_base=api_base, api_key=None, model="deepseek-v4-pro", optional_params={}, litellm_params={}
)
result, route = await _count(
httpx.Response(200, json={"input_tokens": 91}),
count_route=f"{messages_url}/count_tokens",
litellm_params={"api_key": "sk-deepseek-test", **({} if api_base is None else {"api_base": api_base})},
)
assert route.calls.last.request.headers["x-api-key"] == "sk-deepseek-test"
assert result is not None
assert (result.total_tokens, result.error) == (91, False)
@pytest.mark.asyncio
async def test_count_without_a_key_sends_no_api_key_header() -> None:
_, route = await _count(httpx.Response(200, json={"input_tokens": 91}), litellm_params={})
assert "x-api-key" not in route.calls.last.request.headers
@pytest.mark.asyncio
@pytest.mark.parametrize("messages_format", ("anthropic", "openai"))
async def test_count_sends_tools_without_the_custom_type_deepseek_refuses(
messages_format: TokenCountMessagesFormat,
) -> None:
tool: Final[dict[str, JsonValue]] = (
{**_GET_WEATHER, "type": "custom"}
if messages_format == "anthropic"
else {
"type": "function",
"function": {"name": "get_weather", "parameters": _GET_WEATHER["input_schema"]},
}
)
_, route = await _count(
httpx.Response(200, json={"input_tokens": 91}), messages_format=messages_format, tools=[tool]
)
assert json.loads(route.calls.last.request.content)["tools"] == [_GET_WEATHER]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("chat_params", "expected_options"),
(
({}, {}),
({"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}),
({"reasoning_effort": "max"}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "max"}}),
({"reasoning_effort": "unlisted"}, {"thinking": {"type": "enabled"}}),
({"thinking": {"type": "enabled", "budget_tokens": 1024}}, {"thinking": {"type": "enabled"}}),
),
)
async def test_chat_count_maps_thinking_and_effort_the_way_deepseek_chat_inference_does(
chat_params: Mapping[str, JsonValue], expected_options: Mapping[str, JsonValue]
) -> None:
_, route = await _count(
httpx.Response(200, json={"input_tokens": 91}), messages_format="openai", chat_params=chat_params
)
sent: Final = json.loads(route.calls.last.request.content)
assert {name: sent[name] for name in ("thinking", "output_config") if name in sent} == expected_options
@pytest.mark.asyncio
async def test_messages_count_sends_the_thinking_messages_inference_sends() -> None:
_, route = await _count(
httpx.Response(200, json={"input_tokens": 91}),
count_options={"thinking": {"type": "disabled"}, "tool_choice": {"type": "any"}},
)
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["thinking"], sent["tool_choice"]) == ({"type": "disabled"}, {"type": "any"})
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code", (404, 405))
async def test_a_missing_count_route_leaves_the_count_to_the_estimate(status_code: int) -> None:
result, route = await _count(httpx.Response(status_code, text="no such route"))
assert route.called
assert result is None
@pytest.mark.asyncio
async def test_a_rejected_count_answers_with_deepseeks_status() -> None:
result, _ = await _count(httpx.Response(401, json={"error": {"message": "Authentication Fails"}}))
assert result is not None
assert (result.error, result.status_code, result.total_tokens) == (True, 401, 0)
assert "Authentication Fails" in (result.error_message or "")
@pytest.mark.asyncio
async def test_count_without_messages_leaves_the_count_to_the_estimate() -> None:
result, route = await _count(httpx.Response(200, json={"input_tokens": 91}), messages=[])
assert (result, route.called) == (None, False)
@pytest.mark.asyncio
async def test_messages_count_rejects_an_effort_messages_inference_rejects_without_calling_deepseek() -> None:
options: Final[dict[str, JsonValue]] = {"output_config": {"effort": "unlisted"}}
with pytest.raises(AnthropicError) as rejected:
DeepSeekAnthropicMessagesConfig().transform_anthropic_messages_request(
model="deepseek-v4-pro",
messages=[dict(message) for message in _WEATHER_IN_SF],
anthropic_messages_optional_request_params={**options, "max_tokens": 4096},
litellm_params=GenericLiteLLMParams(),
headers={},
)
result, route = await _count(httpx.Response(200, json={"input_tokens": 91}), count_options=options)
assert route.called is False
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
assert (rejected.value.status_code, str(rejected.value.message)) == (400, result.error_message)
@pytest.mark.asyncio
async def test_chat_count_rejects_a_tool_call_chat_inference_rejects_without_calling_deepseek() -> None:
messages: Final[list[dict[str, JsonValue]]] = [
*_WEATHER_IN_SF,
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
result, route = await _count(
httpx.Response(200, json={"input_tokens": 91}), messages_format="openai", messages=messages
)
assert route.called is False
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)

View file

@ -0,0 +1,14 @@
import pytest
import litellm
def test_deepseek_model_listing_with_a_provider_check_lists_the_cost_map_models_offline(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek-test")
listed = litellm.get_valid_models(custom_llm_provider="deepseek", check_provider_endpoint=True)
assert listed
assert sorted(listed) == sorted(litellm.models_by_provider["deepseek"])

View file

@ -31,7 +31,7 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
@pytest.mark.asyncio
@ -1839,9 +1839,9 @@ _PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = {
}
async def _partner_counted_bodies(
model: str, chat_params: dict[str, JsonValue] | None = None
) -> tuple[dict[str, object], ...]:
async def _partner_count(
model: str, messages: list[dict[str, object]], chat_params: dict[str, JsonValue] | None = None
) -> tuple[TokenCountResponse | None, tuple[dict[str, object], ...]]:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
class FakeResponse:
@ -1870,16 +1870,43 @@ async def _partner_counted_bodies(
):
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use=model,
messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS),
messages=copy.deepcopy(messages),
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=[_PARTNER_RESPONSES_TOOL],
chat_params=chat_params or {},
)
return result, fake_http_client.posted_bodies
async def _partner_counted_bodies(
model: str, chat_params: dict[str, JsonValue] | None = None
) -> tuple[dict[str, object], ...]:
result, bodies = await _partner_count(model, _PARTNER_CHAT_TOOL_TURNS, chat_params)
assert result is not None
assert result.total_tokens == 37
return fake_http_client.posted_bodies
return bodies
@pytest.mark.asyncio
async def test_vertex_claude_count_rejects_a_tool_call_chat_inference_rejects_without_calling_vertex() -> None:
unparseable_arguments: Final = [
_PARTNER_CHAT_TOOL_TURNS[0],
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}}
],
},
_PARTNER_CHAT_TOOL_TURNS[2],
]
result, bodies = await _partner_count("claude-opus-5-5", unparseable_arguments)
assert bodies == ()
assert result is not None
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
@pytest.mark.asyncio

View file

@ -367,6 +367,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
count_options=None,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "claude-3-sonnet-20240229"
@ -449,6 +450,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
count_options=None,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "gpt-4"
@ -1269,6 +1271,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
count_options=None,
):
raise ProxyException(
message='{"detail":{"message":"Input is too long for requested model."}}',
@ -1328,6 +1331,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
count_options=None,
):
raise ProxyException(
message='{"Message":"Bearer Token has expired"}',
@ -1387,6 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
request_litellm_params=None,
litellm_call_id=None,
client_oauth_token=None,
count_options=None,
):
raise ProxyException(
message="Rate limit exceeded",
@ -1631,6 +1636,7 @@ class _ScriptedCounter(BaseTokenCounter):
output_format: JsonValue = None,
context_management: object = None,
chat_params: object = None,
count_options: object = None,
) -> TokenCountResponse | None:
self.calls.append((model_to_use, messages, tools, system))
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
@ -4159,3 +4165,242 @@ def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newlin
assert line["request_model"] == forged_model
assert "\n" not in message
assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ')
_COUNTED_OPTION_FIELDS: Final = ("thinking", "tool_choice", "output_config")
_DEEPSEEK_COUNT_ROUTE: Final = "https://api.deepseek.com/anthropic/v1/messages/count_tokens"
_CUSTOM_READ_FILE_TOOL: Final[dict[str, JsonValue]] = {**_READ_FILE_TOOL, "type": "custom"}
def _messages_inference_fields(
config: object, model: str, body: Mapping[str, JsonValue], fields: tuple[str, ...]
) -> dict[str, object]:
from litellm.types.router import GenericLiteLLMParams
inference: Final = config.transform_anthropic_messages_request(
model=model,
messages=json.loads(json.dumps(body["messages"])),
anthropic_messages_optional_request_params={
**{key: json.loads(json.dumps(value)) for key, value in body.items() if key not in ("model", "messages")},
"max_tokens": 4096,
},
litellm_params=GenericLiteLLMParams(),
headers={},
)
return {field: inference.get(field) for field in fields}
async def _count_messages_body(
monkeypatch: pytest.MonkeyPatch,
deployment_params: Mapping[str, JsonValue],
body: Mapping[str, JsonValue],
count_route: str,
provider_reply: httpx.Response,
) -> tuple[object, respx.Route]:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False)
monkeypatch.delenv("DEEPSEEK_ANTHROPIC_API_BASE", raising=False)
monkeypatch.setattr(
proxy_server,
"llm_router",
Router(model_list=[{"model_name": "counted", "litellm_params": dict(deployment_params)}]),
)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post(count_route).mock(return_value=provider_reply)
response: Final = await anthropic_count_tokens(
_json_request({"model": "counted", **body}, "python-httpx/0.28.1"), UserAPIKeyAuth()
)
return response, route
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "options"),
(
("claude-sonnet-4-5", {"tool_choice": {"type": "any"}}),
("claude-sonnet-4-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}}),
("claude-haiku-5-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}, "tool_choice": {"type": "auto"}}),
("claude-opus-5-5", {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}),
),
)
async def test_messages_count_tokens_counts_the_thinking_tool_choice_and_effort_messages_inference_sends(
monkeypatch: pytest.MonkeyPatch, model: str, options: Mapping[str, JsonValue]
) -> None:
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
body: Final[dict[str, JsonValue]] = {
"messages": [{"role": "user", "content": "Read the README"}],
"tools": [_READ_FILE_TOOL],
**options,
}
response, route = await _count_messages_body(
monkeypatch,
{"model": f"anthropic/{model}", "api_key": "sk-ant-test"},
body,
"https://api.anthropic.com/v1/messages/count_tokens",
httpx.Response(200, json={"input_tokens": 120}),
)
sent: Final = json.loads(route.calls.last.request.content)
assert response == {"input_tokens": 120, "litellm_count_source": "provider"}
assert {field: sent.get(field) for field in _COUNTED_OPTION_FIELDS} == _messages_inference_fields(
AnthropicMessagesConfig(), model, body, _COUNTED_OPTION_FIELDS
)
@pytest.mark.asyncio
async def test_messages_count_tokens_leaves_out_an_option_the_request_drops(monkeypatch: pytest.MonkeyPatch) -> None:
response, route = await _count_messages_body(
monkeypatch,
{"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-ant-test"},
{
"messages": [{"role": "user", "content": "Read the README"}],
"tools": [_READ_FILE_TOOL],
"tool_choice": {"type": "any"},
"thinking": {"type": "enabled", "budget_tokens": 2048},
"additional_drop_params": ["thinking"],
},
"https://api.anthropic.com/v1/messages/count_tokens",
httpx.Response(200, json={"input_tokens": 120}),
)
sent: Final = json.loads(route.calls.last.request.content)
assert response == {"input_tokens": 120, "litellm_count_source": "provider"}
assert (sent["tool_choice"], "thinking" in sent) == ({"type": "any"}, False)
@pytest.mark.asyncio
async def test_messages_count_tokens_answers_400_for_a_thinking_budget_messages_inference_rejects(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
body: Final[dict[str, JsonValue]] = {
"messages": [{"role": "user", "content": "Read the README"}],
"thinking": {"type": "enabled", "budget_tokens": "lots"},
}
with pytest.raises(BadRequestError) as rejected:
_messages_inference_fields(AnthropicMessagesConfig(), "claude-opus-5-5", body, _COUNTED_OPTION_FIELDS)
response, route = await _count_messages_body(
monkeypatch,
{"model": "anthropic/claude-opus-5-5", "api_key": "sk-ant-test"},
body,
"https://api.anthropic.com/v1/messages/count_tokens",
httpx.Response(200, json={"input_tokens": 120}),
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert rejected.value.message in json.loads(response.body)["error"]["message"]
assert not route.called
@pytest.mark.asyncio
async def test_messages_count_tokens_counts_deepseek_on_its_anthropic_count_route(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig
body: Final[dict[str, JsonValue]] = {
"messages": [{"role": "user", "content": "Read the README"}],
"tools": [_CUSTOM_READ_FILE_TOOL],
"tool_choice": {"type": "any"},
"thinking": {"type": "enabled"},
"output_config": {"effort": "max"},
}
response, route = await _count_messages_body(
monkeypatch,
{"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test"},
body,
_DEEPSEEK_COUNT_ROUTE,
httpx.Response(200, json={"input_tokens": 91}),
)
sent_request: Final = route.calls.last.request
sent: Final = json.loads(sent_request.content)
inference: Final = _messages_inference_fields(
DeepSeekAnthropicMessagesConfig(), "deepseek-v4-pro", body, ("tool_choice", "output_config", "tools", "thinking")
)
assert response == {"input_tokens": 91, "litellm_count_source": "provider"}
assert sent_request.headers["x-api-key"] == "sk-deepseek-test"
assert (sent["tool_choice"], sent.get("output_config"), sent["tools"], sent["thinking"]["type"]) == (
inference["tool_choice"],
inference["output_config"],
inference["tools"],
inference["thinking"]["type"],
)
@pytest.mark.asyncio
async def test_messages_count_tokens_on_deepseek_estimates_when_the_count_route_is_missing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response, route = await _count_messages_body(
monkeypatch,
{"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test", "api_base": "https://gateway.example"},
{"messages": [{"role": "user", "content": "Read the README"}]},
"https://gateway.example/anthropic/v1/messages/count_tokens",
httpx.Response(404, json={"error": "not found"}),
)
assert route.called
assert isinstance(response, dict)
assert response["litellm_count_source"] == "local_estimate"
assert response["input_tokens"] > 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
("reasoning", "expected_options"),
(
({}, {}),
({"reasoning": {"effort": "none"}}, {"thinking": {"type": "disabled"}}),
({"reasoning": {"effort": "high"}}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "high"}}),
({"reasoning": {"effort": "max"}}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "max"}}),
),
)
async def test_responses_count_on_deepseek_maps_reasoning_effort_the_way_deepseek_chat_inference_does(
monkeypatch: pytest.MonkeyPatch, reasoning: Mapping[str, JsonValue], expected_options: Mapping[str, JsonValue]
) -> None:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False)
monkeypatch.delenv("DEEPSEEK_ANTHROPIC_API_BASE", raising=False)
monkeypatch.setattr(
proxy_server,
"llm_router",
Router(
model_list=[
{
"model_name": "deepseek-counted",
"litellm_params": {"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test"},
}
]
),
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post(_DEEPSEEK_COUNT_ROUTE).mock(return_value=httpx.Response(200, json={"input_tokens": 91}))
result: Final = await count_request_tokens(
request=TokenCountRequest(model="deepseek-counted", messages=[{"role": "user", "content": "Say hi."}]),
call_endpoint=True,
responses_request={
"model": "deepseek-counted",
"input": "Say hi.",
"tools": [_WEATHER_TOOL],
**reasoning,
},
)
sent: Final = json.loads(route.calls.last.request.content)
assert (result.total_tokens, result.count_source) == (91, "provider")
assert {field: sent[field] for field in ("thinking", "output_config") if field in sent} == expected_options
assert all("type" not in tool for tool in sent["tools"])