mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): count thinking, tool_choice, and effort the way inference sends them, and count DeepSeek on its own count route
/v1/messages/count_tokens dropped the body's thinking, tool_choice, and output_config, so a Claude count with tool_choice any came out lower than the bill. The route now passes those options to the provider counter, which runs them through the deployment's own /v1/messages transform first, so the count sees what inference sends. A body inference would reject now answers the same 400 before any provider call. DeepSeek had no provider counter and fell back to a local estimate that ignores thinking. It now counts on DeepSeek's Anthropic-compatible count_tokens route with the same base, key, tool sanitizer, and reasoning mapping inference uses, and falls back to the estimate only when that route is missing.
This commit is contained in:
parent
dff474e450
commit
493cf79031
22 changed files with 1165 additions and 147 deletions
|
|
@ -2,7 +2,7 @@
|
|||
Anthropic Token Counter implementation using the CountTokens API.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
|
|
@ -11,11 +11,13 @@ from pydantic import JsonValue
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountFields,
|
||||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
messages_count_options,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
|
@ -64,12 +66,11 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
Returns:
|
||||
TokenCountResponse with token count, or None if counting fails
|
||||
"""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo
|
||||
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
|
||||
match translated := await anthropic_count_fields(
|
||||
model_to_use, messages, system, tools, messages_format, AnthropicConfig(), chat_params
|
||||
|
|
@ -77,24 +78,51 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
case ChatCountTranslationError():
|
||||
return translated.response(request_model, model_to_use, "anthropic_api")
|
||||
case AnthropicCountFields():
|
||||
return await self.count_tokens(
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=[dict(message) for message in translated.messages],
|
||||
contents=contents,
|
||||
messages=translated.messages,
|
||||
deployment=deployment,
|
||||
request_model=request_model,
|
||||
tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None,
|
||||
tools=translated.tools,
|
||||
system=translated.system,
|
||||
messages_format="anthropic",
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
count_options=translated.options,
|
||||
)
|
||||
case None:
|
||||
pass
|
||||
|
||||
deployment = deployment or {}
|
||||
litellm_params: Final = deployment.get("litellm_params", {})
|
||||
match options := await asyncify(messages_count_options)(
|
||||
AnthropicMessagesConfig(),
|
||||
model_to_use,
|
||||
messages,
|
||||
count_options,
|
||||
(deployment or {}).get("litellm_params", {}),
|
||||
):
|
||||
case ChatCountTranslationError():
|
||||
return options.response(request_model, model_to_use, "anthropic_api")
|
||||
case _:
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
deployment=deployment,
|
||||
request_model=request_model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
count_options=options,
|
||||
)
|
||||
|
||||
async def _count(
|
||||
self,
|
||||
model_to_use: str,
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
deployment: Mapping[str, Any] | None,
|
||||
request_model: str,
|
||||
tools: Sequence[Mapping[str, JsonValue]] | None,
|
||||
system: JsonValue,
|
||||
count_options: Mapping[str, JsonValue],
|
||||
) -> TokenCountResponse | None:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo
|
||||
|
||||
litellm_params: Final = (deployment or {}).get("litellm_params", {})
|
||||
api_base: Final = litellm_params.get("api_base")
|
||||
|
||||
try:
|
||||
|
|
@ -125,10 +153,10 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
|
||||
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
|
||||
model=model_to_use,
|
||||
messages=messages,
|
||||
messages=[dict(message) for message in messages],
|
||||
auth_header=auth_header,
|
||||
api_base=api_base,
|
||||
tools=tools,
|
||||
tools=None if tools is None else [dict(tool) for tool in tools],
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ This module handles the transformation of requests to Anthropic's CountTokens AP
|
|||
import copy
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
|
|
@ -19,8 +21,9 @@ from typing import (
|
|||
from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
anthropic_system_blocks,
|
||||
split_leading_system_run,
|
||||
|
|
@ -29,9 +32,16 @@ from litellm.litellm_core_utils.url_utils import join_api_base
|
|||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
|
||||
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_COUNTED_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_TRANSFORMED_REQUEST: Final = TypeAdapter(dict[str, object])
|
||||
_DROPPED_PATHS: Final = TypeAdapter(tuple[str, ...])
|
||||
_SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue])
|
||||
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
|
||||
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
|
||||
|
|
@ -325,6 +335,64 @@ def anthropic_fields_for_chat_tool_use(
|
|||
)
|
||||
|
||||
|
||||
def _max_output_tokens(model: str, custom_llm_provider: str | None) -> int:
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
try:
|
||||
max_output_tokens: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider).get(
|
||||
"max_output_tokens"
|
||||
)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises a plain Exception for a model the cost map lacks
|
||||
return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
return (
|
||||
max_output_tokens
|
||||
if isinstance(max_output_tokens, int) and max_output_tokens > 0
|
||||
else DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
)
|
||||
|
||||
|
||||
def without_dropped_paths(options: Mapping[str, JsonValue], additional_drop_params: object) -> Mapping[str, JsonValue]:
|
||||
return _COUNT_REQUEST.validate_python(
|
||||
reduce(delete_nested_value, _DROPPED_PATHS.validate_python(additional_drop_params or ()), {**options})
|
||||
)
|
||||
|
||||
|
||||
def messages_count_options(
|
||||
translator: "AnthropicMessagesConfig",
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
options: Mapping[str, JsonValue],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Mapping[str, JsonValue] | ChatCountTranslationError:
|
||||
"""The thinking, tool_choice, and output_config the deployment's /v1/messages transform sends upstream
|
||||
for these Anthropic Messages body options. The transform needs max_tokens, which a count request lacks,
|
||||
so it gets the model's max output tokens: a thinking budget's size never changes a count, only whether
|
||||
thinking stays on."""
|
||||
if not options:
|
||||
return MappingProxyType({})
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
|
||||
try:
|
||||
request: Final = _TRANSFORMED_REQUEST.validate_python(
|
||||
translator.transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=_COUNTED_MESSAGES.validate_python(messages),
|
||||
anthropic_messages_optional_request_params={
|
||||
**without_dropped_paths(options, litellm_params.get("additional_drop_params")),
|
||||
"max_tokens": _max_output_tokens(model, translator.custom_llm_provider),
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams.model_validate({"drop_params": litellm_params.get("drop_params")}),
|
||||
headers={},
|
||||
)
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
return _COUNT_REQUEST.validate_python(
|
||||
{name: request[name] for name in COUNT_TOKEN_OPTION_NAMES if request.get(name) is not None}
|
||||
)
|
||||
|
||||
|
||||
async def anthropic_count_fields(
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API.
|
|||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
|
|
@ -11,10 +11,12 @@ from pydantic import JsonValue
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountFields,
|
||||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
messages_count_options,
|
||||
)
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
|
||||
AzureAIAnthropicCountTokensHandler,
|
||||
|
|
@ -78,11 +80,10 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
Returns:
|
||||
TokenCountResponse with token count, or None if counting fails
|
||||
"""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
|
||||
from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
|
||||
|
||||
match translated := await anthropic_count_fields(
|
||||
|
|
@ -91,24 +92,51 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
case ChatCountTranslationError():
|
||||
return translated.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
||||
case AnthropicCountFields():
|
||||
return await self.count_tokens(
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=[dict(message) for message in translated.messages],
|
||||
contents=contents,
|
||||
messages=translated.messages,
|
||||
deployment=deployment,
|
||||
request_model=request_model,
|
||||
tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None,
|
||||
tools=translated.tools,
|
||||
system=translated.system,
|
||||
messages_format="anthropic",
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
count_options=translated.options,
|
||||
)
|
||||
case None:
|
||||
pass
|
||||
|
||||
deployment = deployment or {}
|
||||
litellm_params: Final = deployment.get("litellm_params", {})
|
||||
match options := await asyncify(messages_count_options)(
|
||||
AzureAnthropicMessagesConfig(),
|
||||
model_to_use,
|
||||
messages,
|
||||
count_options,
|
||||
(deployment or {}).get("litellm_params", {}),
|
||||
):
|
||||
case ChatCountTranslationError():
|
||||
return options.response(request_model, model_to_use, "azure_ai_anthropic_api")
|
||||
case _:
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
deployment=deployment,
|
||||
request_model=request_model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
count_options=options,
|
||||
)
|
||||
|
||||
async def _count(
|
||||
self,
|
||||
model_to_use: str,
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
deployment: Mapping[str, Any] | None,
|
||||
request_model: str,
|
||||
tools: Sequence[Mapping[str, JsonValue]] | None,
|
||||
system: object | None,
|
||||
count_options: Mapping[str, JsonValue],
|
||||
) -> TokenCountResponse | None:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
|
||||
litellm_params: Final = (deployment or {}).get("litellm_params", {})
|
||||
|
||||
# Get Azure AI API key from deployment config or environment
|
||||
api_key = litellm_params.get("api_key")
|
||||
|
|
@ -128,11 +156,11 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
try:
|
||||
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
|
||||
model=model_to_use,
|
||||
messages=messages,
|
||||
messages=[dict(message) for message in messages],
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
tools=tools,
|
||||
tools=None if tools is None else [dict(tool) for tool in tools],
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ class BaseTokenCounter(ABC):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -143,6 +143,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
if not messages:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completi
|
|||
"""
|
||||
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_content_list_to_str,
|
||||
extract_search_results_text,
|
||||
|
|
@ -20,6 +22,35 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
|||
SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
|
||||
|
||||
|
||||
def _thinking_type(thinking: object, reasoning_effort: object) -> str | None:
|
||||
requested: Final = as_str_mapping(thinking)
|
||||
requested_type: Final = None if requested is None else requested.get("type")
|
||||
if isinstance(requested_type, str) and requested_type in ("enabled", "disabled"):
|
||||
return requested_type
|
||||
if reasoning_effort is None:
|
||||
return None
|
||||
return "disabled" if reasoning_effort == "none" else "enabled"
|
||||
|
||||
|
||||
def deepseek_reasoning_params(thinking: object, reasoning_effort: object) -> Mapping[str, object]:
|
||||
"""DeepSeek takes `{"type": "enabled"}` or `{"type": "disabled"}` with no budget_tokens. A thinking
|
||||
param wins. Otherwise `reasoning_effort="none"` turns thinking off and any other effort turns it on,
|
||||
and a supported effort is sent only while thinking is on.
|
||||
|
||||
Reference: https://api-docs.deepseek.com/guides/thinking_mode
|
||||
"""
|
||||
thinking_type: Final = _thinking_type(thinking, reasoning_effort)
|
||||
if thinking_type is None:
|
||||
return MappingProxyType({})
|
||||
if (
|
||||
thinking_type == "enabled"
|
||||
and isinstance(reasoning_effort, str)
|
||||
and reasoning_effort in SUPPORTED_REASONING_EFFORTS
|
||||
):
|
||||
return MappingProxyType({"thinking": {"type": thinking_type}, "reasoning_effort": reasoning_effort})
|
||||
return MappingProxyType({"thinking": {"type": thinking_type}})
|
||||
|
||||
|
||||
class DeepSeekChatConfig(OpenAIGPTConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
|
|
@ -36,40 +67,10 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Map OpenAI params to DeepSeek params.
|
||||
|
||||
Handles `thinking` and `reasoning_effort` parameters for DeepSeek reasoner models.
|
||||
DeepSeek supports `{"type": "enabled"}` and `{"type": "disabled"}` - no budget_tokens
|
||||
like Anthropic. `reasoning_effort="none"` is the OpenAI-style way to ask for thinking
|
||||
off, so it maps to `{"type": "disabled"}`; any other effort keeps thinking on.
|
||||
|
||||
Reference: https://api-docs.deepseek.com/guides/thinking_mode
|
||||
"""
|
||||
# Let parent handle standard params first
|
||||
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
|
||||
|
||||
# Pop thinking/reasoning_effort from optional_params first (parent may have added them)
|
||||
# Then re-add only if valid for DeepSeek
|
||||
thinking_value: Final = optional_params.pop("thinking", None)
|
||||
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
|
||||
|
||||
# Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
|
||||
if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):
|
||||
optional_params["thinking"] = {"type": thinking_value["type"]}
|
||||
|
||||
# Otherwise fall back to reasoning_effort: "none" disables, anything else enables
|
||||
elif reasoning_effort is not None:
|
||||
optional_params["thinking"] = {"type": "disabled" if reasoning_effort == "none" else "enabled"}
|
||||
|
||||
if (
|
||||
isinstance(reasoning_effort, str)
|
||||
and reasoning_effort in SUPPORTED_REASONING_EFFORTS
|
||||
and optional_params["thinking"] == {"type": "enabled"}
|
||||
):
|
||||
return {**optional_params, "reasoning_effort": reasoning_effort}
|
||||
|
||||
return optional_params
|
||||
return {**optional_params, **deepseek_reasoning_params(thinking_value, reasoning_effort)}
|
||||
|
||||
def _fill_reasoning_content(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
|
||||
"""
|
||||
|
|
|
|||
46
litellm/llms/deepseek/common_utils.py
Normal file
46
litellm/llms/deepseek/common_utils.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig, deepseek_anthropic_base
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
class DeepSeekModelInfo(BaseLLMModelInfo):
|
||||
def get_models(
|
||||
self, api_key: str | None = None, api_base: str | None = None
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
from litellm.utils import get_valid_models
|
||||
|
||||
return get_valid_models(custom_llm_provider=LlmProviders.DEEPSEEK.value)
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return DeepSeekAnthropicMessagesConfig.get_api_key(api_key)
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
return deepseek_anthropic_base(api_base)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: matches validate_environment's dict contract shared by every provider override
|
||||
resolved_api_key: Final = self.get_api_key(api_key)
|
||||
return {**headers, **({} if resolved_api_key is None else {"x-api-key": resolved_api_key})}
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> str | None:
|
||||
return model.removeprefix(f"{LlmProviders.DEEPSEEK.value}/")
|
||||
|
||||
def get_token_counter(self) -> BaseTokenCounter | None:
|
||||
from litellm.llms.deepseek.count_tokens.token_counter import DeepSeekTokenCounter
|
||||
|
||||
return DeepSeekTokenCounter()
|
||||
211
litellm/llms/deepseek/count_tokens/token_counter.py
Normal file
211
litellm/llms/deepseek/count_tokens/token_counter.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
"""
|
||||
DeepSeek token counting through the count_tokens route of its Anthropic-compatible API.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountFields,
|
||||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
messages_count_options,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.deepseek.chat.transformation import deepseek_reasoning_params
|
||||
from litellm.llms.deepseek.common_utils import DeepSeekModelInfo
|
||||
from litellm.llms.deepseek.messages.transformation import (
|
||||
DeepSeekAnthropicMessagesConfig,
|
||||
sanitize_tools_for_deepseek,
|
||||
)
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
DEEPSEEK_TOKENIZER_TYPE: Final = "deepseek_api"
|
||||
_CHAT_REASONING_PARAMS: Final = ("thinking", "reasoning_effort")
|
||||
_COUNT_ROUTE_MISSING: Final = frozenset({404, 405})
|
||||
_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_OPTIONS: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
deepseek_count_tokens_handler: Final = AnthropicCountTokensHandler()
|
||||
|
||||
|
||||
def _chat_reasoning_options(chat_params: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
|
||||
mapped: Final = deepseek_reasoning_params(chat_params.get("thinking"), chat_params.get("reasoning_effort"))
|
||||
effort: Final = mapped.get("reasoning_effort")
|
||||
return _OPTIONS.validate_python(
|
||||
{
|
||||
**({} if mapped.get("thinking") is None else {"thinking": mapped["thinking"]}),
|
||||
**({} if effort is None else {"output_config": {"effort": effort}}),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _with_chat_reasoning(
|
||||
options: Mapping[str, JsonValue], reasoning: Mapping[str, JsonValue]
|
||||
) -> Mapping[str, JsonValue]:
|
||||
output_config: Final = options.get("output_config")
|
||||
reasoning_output_config: Final = reasoning.get("output_config")
|
||||
return _OPTIONS.validate_python(
|
||||
{
|
||||
**options,
|
||||
**reasoning,
|
||||
**(
|
||||
{"output_config": {**output_config, **reasoning_output_config}}
|
||||
if isinstance(output_config, dict) and isinstance(reasoning_output_config, dict)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _litellm_params(deployment: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
params: Final = None if deployment is None else as_str_mapping(deployment.get("litellm_params"))
|
||||
return MappingProxyType({}) if params is None else params
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
class DeepSeekTokenCounter(BaseTokenCounter):
|
||||
"""Counts with the request DeepSeek bills: an Anthropic Messages body sent to the Anthropic-compatible
|
||||
count_tokens route, with thinking and effort mapped the way the deployment's inference maps them."""
|
||||
|
||||
def __init__(self, handler: AnthropicCountTokensHandler = deepseek_count_tokens_handler) -> None:
|
||||
self._handler: Final = handler
|
||||
|
||||
def should_use_token_counting_api(self, custom_llm_provider: str | None = None) -> bool:
|
||||
return custom_llm_provider == LlmProviders.DEEPSEEK.value
|
||||
|
||||
async def count_tokens(
|
||||
self,
|
||||
model_to_use: str,
|
||||
messages: Sequence[Mapping[str, JsonValue]] | None,
|
||||
contents: Sequence[Mapping[str, object]] | None,
|
||||
deployment: Mapping[str, object] | None = None,
|
||||
request_model: str = "",
|
||||
tools: Sequence[Mapping[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
messages_format: TokenCountMessagesFormat = "anthropic",
|
||||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
litellm_params: Final = _litellm_params(deployment)
|
||||
match translated := await anthropic_count_fields(
|
||||
model_to_use,
|
||||
messages,
|
||||
system,
|
||||
tools,
|
||||
messages_format,
|
||||
AnthropicConfig(),
|
||||
{name: value for name, value in chat_params.items() if name not in _CHAT_REASONING_PARAMS},
|
||||
):
|
||||
case ChatCountTranslationError():
|
||||
return translated.response(request_model, model_to_use, DEEPSEEK_TOKENIZER_TYPE)
|
||||
case AnthropicCountFields():
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=translated.messages,
|
||||
litellm_params=litellm_params,
|
||||
request_model=request_model,
|
||||
tools=translated.tools,
|
||||
system=translated.system,
|
||||
count_options=_with_chat_reasoning(translated.options, _chat_reasoning_options(chat_params)),
|
||||
)
|
||||
case None:
|
||||
pass
|
||||
|
||||
if messages_format == "openai":
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
litellm_params=litellm_params,
|
||||
request_model=request_model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
count_options=_chat_reasoning_options(chat_params),
|
||||
)
|
||||
|
||||
match options := await asyncify(messages_count_options)(
|
||||
DeepSeekAnthropicMessagesConfig(), model_to_use, messages, count_options, litellm_params
|
||||
):
|
||||
case ChatCountTranslationError():
|
||||
return options.response(request_model, model_to_use, DEEPSEEK_TOKENIZER_TYPE)
|
||||
case _:
|
||||
return await self._count(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
litellm_params=litellm_params,
|
||||
request_model=request_model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
count_options=options,
|
||||
)
|
||||
|
||||
async def _count(
|
||||
self,
|
||||
model_to_use: str,
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
litellm_params: Mapping[str, object],
|
||||
request_model: str,
|
||||
tools: Sequence[Mapping[str, JsonValue]] | None,
|
||||
system: JsonValue,
|
||||
count_options: Mapping[str, JsonValue],
|
||||
) -> TokenCountResponse | None:
|
||||
model_info: Final = DeepSeekModelInfo()
|
||||
try:
|
||||
result: Final = await self._handler.handle_count_tokens_request(
|
||||
model=model_to_use,
|
||||
messages=[dict(message) for message in messages],
|
||||
auth_header=model_info.validate_environment(
|
||||
headers={},
|
||||
model=model_to_use,
|
||||
messages=(),
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
api_key=_optional_str(litellm_params.get("api_key")),
|
||||
),
|
||||
api_base=model_info.get_api_base(_optional_str(litellm_params.get("api_base"))),
|
||||
tools=None if tools is None else _TOOLS.validate_python(sanitize_tools_for_deepseek(tools)),
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
)
|
||||
except AnthropicError as e:
|
||||
if e.status_code in _COUNT_ROUTE_MISSING:
|
||||
verbose_logger.debug(
|
||||
"DeepSeek count_tokens route answered %s, so the count is estimated", e.status_code
|
||||
)
|
||||
return None
|
||||
verbose_logger.warning("DeepSeek CountTokens error: status=%s, message=%s", e.status_code, e.message)
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
request_model=request_model,
|
||||
model_used=model_to_use,
|
||||
tokenizer_type=DEEPSEEK_TOKENIZER_TYPE,
|
||||
error=True,
|
||||
error_message=e.message,
|
||||
status_code=e.status_code,
|
||||
)
|
||||
input_tokens: Final = result.get("input_tokens")
|
||||
return TokenCountResponse(
|
||||
total_tokens=input_tokens if isinstance(input_tokens, int) else 0,
|
||||
request_model=request_model,
|
||||
model_used=model_to_use,
|
||||
tokenizer_type=DEEPSEEK_TOKENIZER_TYPE,
|
||||
original_response=result,
|
||||
)
|
||||
|
|
@ -16,6 +16,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_TOOL_LIST: Final = TypeAdapter(list[object])
|
||||
_REQUEST_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_TOOL_FIELDS: Final = TypeAdapter(dict[object, object])
|
||||
|
||||
|
||||
|
|
@ -34,6 +35,20 @@ def _sanitized_tool(tool: object) -> object:
|
|||
return tool if stripped is None else stripped
|
||||
|
||||
|
||||
def sanitize_tools_for_deepseek(tools: Sequence[object]) -> Sequence[object]:
|
||||
return [_sanitized_tool(tool) for tool in tools]
|
||||
|
||||
|
||||
def deepseek_anthropic_base(api_base: str | None) -> str:
|
||||
configured: Final = DeepSeekAnthropicMessagesConfig.get_api_base(api_base=api_base).rstrip("/")
|
||||
if configured.endswith("/v1/messages") and "/anthropic/" in configured:
|
||||
return configured.removesuffix("/v1/messages")
|
||||
stripped: Final = configured.removesuffix("/v1/messages").removesuffix("/v1").removesuffix("/beta")
|
||||
if stripped.endswith("/anthropic") or "/anthropic/" in stripped:
|
||||
return stripped
|
||||
return join_api_base(stripped, "anthropic")
|
||||
|
||||
|
||||
class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
|
||||
"""
|
||||
DeepSeek exposes an Anthropic-compatible Messages API at
|
||||
|
|
@ -102,25 +117,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
|
|||
litellm_params: dict,
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
base_url = self.get_api_base(api_base=api_base).rstrip("/")
|
||||
|
||||
if base_url.endswith("/v1/messages") and "/anthropic/" in base_url:
|
||||
return base_url
|
||||
base_url = base_url.removesuffix("/v1/messages")
|
||||
base_url = base_url.removesuffix("/v1")
|
||||
base_url = base_url.removesuffix("/beta")
|
||||
|
||||
if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url:
|
||||
base_url = join_api_base(base_url, "anthropic")
|
||||
|
||||
return join_api_base(base_url, "v1/messages")
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_tools_for_deepseek(tools: object) -> object:
|
||||
if not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)]
|
||||
return join_api_base(deepseek_anthropic_base(api_base), "v1/messages")
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
|
|
@ -130,13 +127,20 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
|
|||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
anthropic_messages_request: Final = super().transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
anthropic_messages_request: Final = _REQUEST_FIELDS.validate_python(
|
||||
super().transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
)
|
||||
if "tools" in anthropic_messages_request:
|
||||
anthropic_messages_request["tools"] = self._sanitize_tools_for_deepseek(anthropic_messages_request["tools"])
|
||||
return anthropic_messages_request
|
||||
tools: Final = anthropic_messages_request.get("tools")
|
||||
if not isinstance(tools, list):
|
||||
return anthropic_messages_request
|
||||
sanitized: Final = {
|
||||
**anthropic_messages_request,
|
||||
"tools": sanitize_tools_for_deepseek(_TOOL_LIST.validate_python(tools)),
|
||||
}
|
||||
return sanitized
|
||||
|
|
|
|||
|
|
@ -534,6 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse | None:
|
||||
"""
|
||||
Count tokens using OpenAI's Responses API /input_tokens endpoint.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from enum import Enum
|
|||
from functools import lru_cache
|
||||
from itertools import accumulate, chain, repeat, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast, get_type_hints
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
|
@ -33,6 +33,9 @@ from litellm.types.llms.vertex_ai import (
|
|||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.utils import supports_response_schema, supports_system_messages
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountFields
|
||||
|
||||
GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
|
||||
{"fps": "fps", "start_offset": "startOffset", "end_offset": "endOffset"}
|
||||
)
|
||||
|
|
@ -1395,21 +1398,17 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
|
||||
return custom_llm_provider == LlmProviders.VERTEX_AI.value
|
||||
|
||||
async def _count_translated_claude_request(
|
||||
async def _translated_claude_request(
|
||||
self,
|
||||
model_to_use: str,
|
||||
messages: Sequence[Mapping[str, object]] | None,
|
||||
deployment: Mapping[str, object],
|
||||
request_model: str,
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object | None,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
output_format: JsonValue,
|
||||
context_management: CountContextManagement | None,
|
||||
chat_params: Mapping[str, JsonValue],
|
||||
) -> TokenCountResponse | None:
|
||||
) -> "TokenCountResponse | AnthropicCountFields | None":
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountFields,
|
||||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
)
|
||||
|
|
@ -1419,27 +1418,12 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
return None
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
|
||||
|
||||
match translated := await anthropic_count_fields(
|
||||
translated: Final = await anthropic_count_fields(
|
||||
model_to_use, messages or [], system, tools, messages_format, VertexAIAnthropicConfig(), chat_params
|
||||
):
|
||||
case ChatCountTranslationError():
|
||||
return translated.response(request_model, model_to_use, "vertex_ai_partner_models")
|
||||
case AnthropicCountFields():
|
||||
return await self.count_tokens(
|
||||
model_to_use=model_to_use,
|
||||
messages=[dict(message) for message in translated.messages],
|
||||
contents=None,
|
||||
deployment=dict(deployment),
|
||||
request_model=request_model,
|
||||
tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None,
|
||||
system=translated.system,
|
||||
messages_format="anthropic",
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
count_options=translated.options,
|
||||
)
|
||||
case None:
|
||||
return None
|
||||
)
|
||||
if isinstance(translated, ChatCountTranslationError):
|
||||
return translated.response(request_model, model_to_use, "vertex_ai_partner_models")
|
||||
return translated
|
||||
|
||||
async def count_tokens(
|
||||
self,
|
||||
|
|
@ -1493,20 +1477,17 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
|
||||
# Check if this is a partner model (Claude, Mistral, etc.)
|
||||
if VertexAIPartnerModels.is_vertex_partner_model(model_to_use):
|
||||
translated_count: Final = await self._count_translated_claude_request(
|
||||
translated: Final = await self._translated_claude_request(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
deployment=deployment,
|
||||
request_model=request_model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
chat_params=chat_params,
|
||||
)
|
||||
if translated_count is not None:
|
||||
return translated_count
|
||||
if isinstance(translated, TokenCountResponse):
|
||||
return translated
|
||||
|
||||
# Use partner models token counter
|
||||
partner_models_handler: Final = VertexAIPartnerModels()
|
||||
|
|
@ -1531,14 +1512,22 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
try:
|
||||
result = await partner_models_handler.count_tokens(
|
||||
model=model_to_use,
|
||||
messages=messages or [],
|
||||
messages=(
|
||||
(messages or []) if translated is None else [dict(message) for message in translated.messages]
|
||||
),
|
||||
litellm_params=partner_litellm_params,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
system=system,
|
||||
tools=tools,
|
||||
count_options=count_options,
|
||||
system=system if translated is None else translated.system,
|
||||
tools=(
|
||||
tools
|
||||
if translated is None
|
||||
else None
|
||||
if translated.tools is None
|
||||
else [dict(tool) for tool in translated.tools]
|
||||
),
|
||||
count_options=MappingProxyType({}) if translated is None else translated.options,
|
||||
)
|
||||
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
|
||||
status_code: Final = e.response.status_code
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# What is this?
|
||||
## API Handler for calling Vertex AI Partner Models
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -259,13 +259,13 @@ class VertexAIPartnerModels(VertexBase):
|
|||
async def count_tokens(
|
||||
self,
|
||||
model: str,
|
||||
messages: list,
|
||||
litellm_params: dict,
|
||||
vertex_project=None,
|
||||
vertex_location=None,
|
||||
vertex_credentials=None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
litellm_params: Mapping[str, object],
|
||||
vertex_project: object = None,
|
||||
vertex_location: object = None,
|
||||
vertex_credentials: object = None,
|
||||
system: object | None = None,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
tools: Sequence[Mapping[str, object]] | None = None,
|
||||
count_options: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
"""
|
||||
|
|
@ -307,13 +307,18 @@ class VertexAIPartnerModels(VertexBase):
|
|||
}
|
||||
|
||||
# Prepare litellm_params with credentials
|
||||
_litellm_params: Final = litellm_params.copy()
|
||||
if vertex_project:
|
||||
_litellm_params["vertex_project"] = vertex_project
|
||||
if vertex_location:
|
||||
_litellm_params["vertex_location"] = vertex_location
|
||||
if vertex_credentials:
|
||||
_litellm_params["vertex_credentials"] = vertex_credentials
|
||||
_litellm_params: Final = {
|
||||
**litellm_params,
|
||||
**{
|
||||
key: value
|
||||
for key, value in (
|
||||
("vertex_project", vertex_project),
|
||||
("vertex_location", vertex_location),
|
||||
("vertex_credentials", vertex_credentials),
|
||||
)
|
||||
if value
|
||||
},
|
||||
}
|
||||
|
||||
# Call the token counter
|
||||
token_counter: Final = VertexAIPartnerModelsTokenCounter()
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.anthropic_interface.exceptions import (
|
|||
)
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, without_dropped_paths
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
)
|
||||
|
|
@ -342,6 +343,7 @@ class _CountTokensBodyOptions(TypedDict):
|
|||
output_format: ReadOnly[JsonValue]
|
||||
context_management: ReadOnly[JsonValue]
|
||||
request_litellm_params: ReadOnly[Mapping[str, JsonValue]]
|
||||
count_options: ReadOnly[Mapping[str, JsonValue]]
|
||||
|
||||
|
||||
def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions:
|
||||
|
|
@ -352,6 +354,12 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
|
|||
"request_litellm_params": MappingProxyType(
|
||||
{"drop_params": True, **sent_params} if auto_drop_params else sent_params
|
||||
),
|
||||
"count_options": MappingProxyType(
|
||||
without_dropped_paths(
|
||||
{name: data[name] for name in COUNT_TOKEN_OPTION_NAMES if name in data},
|
||||
data.get("additional_drop_params"),
|
||||
)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14814,6 +14814,7 @@ async def _try_provider_token_count(
|
|||
output_format: JsonValue = None,
|
||||
context_management: CountContextManagement | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
"""Attempt provider-specific token counting. Returns None to fall through to local counting."""
|
||||
if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider):
|
||||
|
|
@ -14841,6 +14842,7 @@ async def _try_provider_token_count(
|
|||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
chat_params=chat_params,
|
||||
count_options=count_options,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
return ProviderCountFailure(
|
||||
|
|
@ -15320,6 +15322,7 @@ async def count_request_tokens(
|
|||
responses_request: Mapping[str, object] | None = None,
|
||||
litellm_call_id: str | None = None,
|
||||
client_oauth_token: str | None = None,
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> TokenCountResponse:
|
||||
call_id: Final = resolve_litellm_call_id(litellm_call_id)
|
||||
if request.prompt is None and request.messages is None and request.contents is None:
|
||||
|
|
@ -15339,6 +15342,7 @@ async def count_request_tokens(
|
|||
request_litellm_params=request_litellm_params,
|
||||
responses_request=responses_request,
|
||||
client_oauth_token=client_oauth_token,
|
||||
count_options=count_options,
|
||||
)
|
||||
return await logged_token_count(
|
||||
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
|
||||
|
|
@ -15381,6 +15385,7 @@ async def _count_request_tokens(
|
|||
request_litellm_params: Mapping[str, JsonValue],
|
||||
responses_request: Mapping[str, object] | None,
|
||||
client_oauth_token: str | None,
|
||||
count_options: Mapping[str, JsonValue],
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -15488,6 +15493,7 @@ async def _count_request_tokens(
|
|||
output_format=output_format,
|
||||
context_management=_count_context_management(context_management, request.model, request_litellm_params),
|
||||
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
|
||||
count_options=count_options,
|
||||
),
|
||||
model_to_use=model_to_use,
|
||||
request_model=request.model,
|
||||
|
|
|
|||
|
|
@ -10142,6 +10142,10 @@ class ProviderConfigManager:
|
|||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
|
||||
return AzureFoundryModelInfo(model=model)
|
||||
elif LlmProviders.DEEPSEEK == provider:
|
||||
from litellm.llms.deepseek.common_utils import DeepSeekModelInfo
|
||||
|
||||
return DeepSeekModelInfo()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -16,9 +16,12 @@ import litellm
|
|||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
COUNT_TOKEN_OPTION_NAMES,
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
|
||||
AzureAIAnthropicCountTokensHandler,
|
||||
)
|
||||
|
|
@ -26,6 +29,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
|
|||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
||||
|
|
@ -482,6 +487,8 @@ async def _counted_body(
|
|||
tools: list[dict[str, JsonValue]] | None,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
model: str = "claude-opus-5-5",
|
||||
) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]:
|
||||
sent: Final[SimpleQueue[bytes]] = SimpleQueue()
|
||||
|
||||
|
|
@ -492,14 +499,15 @@ async def _counted_body(
|
|||
with respx.mock:
|
||||
respx.post(count_endpoint).mock(side_effect=count)
|
||||
result: Final = await counter_type().count_tokens(
|
||||
model_to_use="claude-opus-5-5",
|
||||
model_to_use=model,
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
request_model="claude-opus-5-5",
|
||||
request_model=model,
|
||||
tools=tools,
|
||||
messages_format=messages_format,
|
||||
chat_params=chat_params,
|
||||
count_options=count_options,
|
||||
)
|
||||
return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize()))
|
||||
|
||||
|
|
@ -842,3 +850,112 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat
|
|||
inference["tool_choice"],
|
||||
inference["tools"],
|
||||
)
|
||||
|
||||
|
||||
_MESSAGES_CONFIGS: Final = MappingProxyType(
|
||||
{AnthropicTokenCounter: AnthropicMessagesConfig, AzureAIAnthropicTokenCounter: AzureAnthropicMessagesConfig}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
@pytest.mark.parametrize(
|
||||
("model", "options"),
|
||||
(
|
||||
("claude-sonnet-4-5", {"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}),
|
||||
("claude-haiku-5-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}}),
|
||||
("claude-opus-5-5", {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}),
|
||||
("claude-unlisted-model", {"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}),
|
||||
),
|
||||
)
|
||||
async def test_counter_sends_the_thinking_tool_choice_and_effort_messages_inference_sends(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
model: str,
|
||||
options: Mapping[str, JsonValue],
|
||||
) -> None:
|
||||
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
|
||||
inference: Final = _MESSAGES_CONFIGS[counter_type]().transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=deepcopy(messages),
|
||||
anthropic_messages_optional_request_params={**deepcopy(dict(options)), "max_tokens": 4096},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
result, bodies = await _counted_body(
|
||||
counter_type, count_endpoint, messages, None, "anthropic", count_options=options, model=model
|
||||
)
|
||||
|
||||
(body,) = bodies
|
||||
assert result is not None
|
||||
assert result.total_tokens == 31
|
||||
assert {name: body.get(name) for name in COUNT_TOKEN_OPTION_NAMES} == {
|
||||
name: inference.get(name) for name in COUNT_TOKEN_OPTION_NAMES
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_leaves_out_an_option_the_deployment_drops(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
sent: Final[SimpleQueue[bytes]] = SimpleQueue()
|
||||
|
||||
def count(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request.content)
|
||||
return httpx.Response(200, json={"input_tokens": 31})
|
||||
|
||||
with respx.mock:
|
||||
respx.post(count_endpoint).mock(side_effect=count)
|
||||
await counter_type().count_tokens(
|
||||
model_to_use="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
contents=None,
|
||||
deployment={
|
||||
"litellm_params": {
|
||||
"api_key": "sk-ant-api03-test-key",
|
||||
"api_base": "https://gateway.example",
|
||||
"additional_drop_params": ["tool_choice"],
|
||||
}
|
||||
},
|
||||
request_model="claude-sonnet-4-5",
|
||||
messages_format="anthropic",
|
||||
count_options={"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}},
|
||||
)
|
||||
|
||||
body: Final = TypeAdapter(dict[str, JsonValue]).validate_json(sent.get())
|
||||
assert ("tool_choice" in body, body["thinking"]) == (False, {"type": "enabled", "budget_tokens": 2048})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_rejects_a_thinking_budget_messages_inference_rejects_without_calling_anthropic(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
options: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": "lots"}}
|
||||
with pytest.raises(BadRequestError) as rejected:
|
||||
_MESSAGES_CONFIGS[counter_type]().transform_anthropic_messages_request(
|
||||
model="claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
anthropic_messages_optional_request_params={**deepcopy(options), "max_tokens": 4096},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
result, bodies = await _counted_body(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
None,
|
||||
"anthropic",
|
||||
count_options=options,
|
||||
)
|
||||
|
||||
assert bodies == ()
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
assert rejected.value.message in (result.error_message or "")
|
||||
|
|
|
|||
0
tests/unit/llms/deepseek/count_tokens/__init__.py
Normal file
0
tests/unit/llms/deepseek/count_tokens/__init__.py
Normal file
213
tests/unit/llms/deepseek/count_tokens/test_token_counter.py
Normal file
213
tests/unit/llms/deepseek/count_tokens/test_token_counter.py
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import JsonValue
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.deepseek.count_tokens.token_counter import DeepSeekTokenCounter
|
||||
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
_DEFAULT_COUNT_ROUTE: Final = "https://api.deepseek.com/anthropic/v1/messages/count_tokens"
|
||||
_WEATHER_IN_SF: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
|
||||
_GET_WEATHER: Final[dict[str, JsonValue]] = {
|
||||
"name": "get_weather",
|
||||
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _deepseek_env_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for name in ("DEEPSEEK_API_BASE", "DEEPSEEK_ANTHROPIC_API_BASE", "DEEPSEEK_API_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setattr(litellm, "api_key", None)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
|
||||
async def _count(
|
||||
reply: httpx.Response,
|
||||
count_route: str = _DEFAULT_COUNT_ROUTE,
|
||||
litellm_params: Mapping[str, JsonValue] = MappingProxyType({"api_key": "sk-deepseek-test"}),
|
||||
messages_format: TokenCountMessagesFormat = "anthropic",
|
||||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
chat_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
count_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
messages: list[dict[str, JsonValue]] = _WEATHER_IN_SF,
|
||||
) -> tuple[TokenCountResponse | None, respx.Route]:
|
||||
with respx.mock:
|
||||
route: Final = respx.post(count_route).mock(return_value=reply)
|
||||
result: Final = await DeepSeekTokenCounter().count_tokens(
|
||||
model_to_use="deepseek-v4-pro",
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": dict(litellm_params)},
|
||||
request_model="deepseek-pro",
|
||||
tools=tools,
|
||||
messages_format=messages_format,
|
||||
chat_params=chat_params,
|
||||
count_options=count_options,
|
||||
)
|
||||
return result, route
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
(
|
||||
None,
|
||||
"https://api.deepseek.com",
|
||||
"https://api.deepseek.com/beta",
|
||||
"https://api.deepseek.com/anthropic/v1/messages",
|
||||
"https://gateway.example/deepseek",
|
||||
),
|
||||
)
|
||||
async def test_count_posts_under_the_base_messages_inference_posts_to(api_base: str | None) -> None:
|
||||
messages_url: Final = DeepSeekAnthropicMessagesConfig().get_complete_url(
|
||||
api_base=api_base, api_key=None, model="deepseek-v4-pro", optional_params={}, litellm_params={}
|
||||
)
|
||||
|
||||
result, route = await _count(
|
||||
httpx.Response(200, json={"input_tokens": 91}),
|
||||
count_route=f"{messages_url}/count_tokens",
|
||||
litellm_params={"api_key": "sk-deepseek-test", **({} if api_base is None else {"api_base": api_base})},
|
||||
)
|
||||
|
||||
assert route.calls.last.request.headers["x-api-key"] == "sk-deepseek-test"
|
||||
assert result is not None
|
||||
assert (result.total_tokens, result.error) == (91, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_without_a_key_sends_no_api_key_header() -> None:
|
||||
_, route = await _count(httpx.Response(200, json={"input_tokens": 91}), litellm_params={})
|
||||
|
||||
assert "x-api-key" not in route.calls.last.request.headers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("messages_format", ("anthropic", "openai"))
|
||||
async def test_count_sends_tools_without_the_custom_type_deepseek_refuses(
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
) -> None:
|
||||
tool: Final[dict[str, JsonValue]] = (
|
||||
{**_GET_WEATHER, "type": "custom"}
|
||||
if messages_format == "anthropic"
|
||||
else {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": _GET_WEATHER["input_schema"]},
|
||||
}
|
||||
)
|
||||
|
||||
_, route = await _count(
|
||||
httpx.Response(200, json={"input_tokens": 91}), messages_format=messages_format, tools=[tool]
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["tools"] == [_GET_WEATHER]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("chat_params", "expected_options"),
|
||||
(
|
||||
({}, {}),
|
||||
({"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}),
|
||||
({"reasoning_effort": "max"}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "max"}}),
|
||||
({"reasoning_effort": "unlisted"}, {"thinking": {"type": "enabled"}}),
|
||||
({"thinking": {"type": "enabled", "budget_tokens": 1024}}, {"thinking": {"type": "enabled"}}),
|
||||
),
|
||||
)
|
||||
async def test_chat_count_maps_thinking_and_effort_the_way_deepseek_chat_inference_does(
|
||||
chat_params: Mapping[str, JsonValue], expected_options: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
_, route = await _count(
|
||||
httpx.Response(200, json={"input_tokens": 91}), messages_format="openai", chat_params=chat_params
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert {name: sent[name] for name in ("thinking", "output_config") if name in sent} == expected_options
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_count_sends_the_thinking_messages_inference_sends() -> None:
|
||||
_, route = await _count(
|
||||
httpx.Response(200, json={"input_tokens": 91}),
|
||||
count_options={"thinking": {"type": "disabled"}, "tool_choice": {"type": "any"}},
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["thinking"], sent["tool_choice"]) == ({"type": "disabled"}, {"type": "any"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status_code", (404, 405))
|
||||
async def test_a_missing_count_route_leaves_the_count_to_the_estimate(status_code: int) -> None:
|
||||
result, route = await _count(httpx.Response(status_code, text="no such route"))
|
||||
|
||||
assert route.called
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_rejected_count_answers_with_deepseeks_status() -> None:
|
||||
result, _ = await _count(httpx.Response(401, json={"error": {"message": "Authentication Fails"}}))
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.total_tokens) == (True, 401, 0)
|
||||
assert "Authentication Fails" in (result.error_message or "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_without_messages_leaves_the_count_to_the_estimate() -> None:
|
||||
result, route = await _count(httpx.Response(200, json={"input_tokens": 91}), messages=[])
|
||||
|
||||
assert (result, route.called) == (None, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_count_rejects_an_effort_messages_inference_rejects_without_calling_deepseek() -> None:
|
||||
options: Final[dict[str, JsonValue]] = {"output_config": {"effort": "unlisted"}}
|
||||
with pytest.raises(AnthropicError) as rejected:
|
||||
DeepSeekAnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model="deepseek-v4-pro",
|
||||
messages=[dict(message) for message in _WEATHER_IN_SF],
|
||||
anthropic_messages_optional_request_params={**options, "max_tokens": 4096},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
result, route = await _count(httpx.Response(200, json={"input_tokens": 91}), count_options=options)
|
||||
|
||||
assert route.called is False
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
assert (rejected.value.status_code, str(rejected.value.message)) == (400, result.error_message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_count_rejects_a_tool_call_chat_inference_rejects_without_calling_deepseek() -> None:
|
||||
messages: Final[list[dict[str, JsonValue]]] = [
|
||||
*_WEATHER_IN_SF,
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
|
||||
result, route = await _count(
|
||||
httpx.Response(200, json={"input_tokens": 91}), messages_format="openai", messages=messages
|
||||
)
|
||||
|
||||
assert route.called is False
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
14
tests/unit/llms/deepseek/test_deepseek_common_utils.py
Normal file
14
tests/unit/llms/deepseek/test_deepseek_common_utils.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
def test_deepseek_model_listing_with_a_provider_check_lists_the_cost_map_models_offline(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek-test")
|
||||
|
||||
listed = litellm.get_valid_models(custom_llm_provider="deepseek", check_provider_endpoint=True)
|
||||
|
||||
assert listed
|
||||
assert sorted(listed) == sorted(litellm.models_by_provider["deepseek"])
|
||||
|
|
@ -31,7 +31,7 @@ from litellm.llms.vertex_ai.common_utils import (
|
|||
)
|
||||
from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1839,9 +1839,9 @@ _PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = {
|
|||
}
|
||||
|
||||
|
||||
async def _partner_counted_bodies(
|
||||
model: str, chat_params: dict[str, JsonValue] | None = None
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
async def _partner_count(
|
||||
model: str, messages: list[dict[str, object]], chat_params: dict[str, JsonValue] | None = None
|
||||
) -> tuple[TokenCountResponse | None, tuple[dict[str, object], ...]]:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
|
||||
class FakeResponse:
|
||||
|
|
@ -1870,16 +1870,43 @@ async def _partner_counted_bodies(
|
|||
):
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use=model,
|
||||
messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS),
|
||||
messages=copy.deepcopy(messages),
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=[_PARTNER_RESPONSES_TOOL],
|
||||
chat_params=chat_params or {},
|
||||
)
|
||||
return result, fake_http_client.posted_bodies
|
||||
|
||||
|
||||
async def _partner_counted_bodies(
|
||||
model: str, chat_params: dict[str, JsonValue] | None = None
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
result, bodies = await _partner_count(model, _PARTNER_CHAT_TOOL_TURNS, chat_params)
|
||||
assert result is not None
|
||||
assert result.total_tokens == 37
|
||||
return fake_http_client.posted_bodies
|
||||
return bodies
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_rejects_a_tool_call_chat_inference_rejects_without_calling_vertex() -> None:
|
||||
unparseable_arguments: Final = [
|
||||
_PARTNER_CHAT_TOOL_TURNS[0],
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}}
|
||||
],
|
||||
},
|
||||
_PARTNER_CHAT_TOOL_TURNS[2],
|
||||
]
|
||||
|
||||
result, bodies = await _partner_count("claude-opus-5-5", unparseable_arguments)
|
||||
|
||||
assert bodies == ()
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -367,6 +367,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
count_options=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "claude-3-sonnet-20240229"
|
||||
|
|
@ -449,6 +450,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
count_options=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "gpt-4"
|
||||
|
|
@ -1269,6 +1271,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
count_options=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"detail":{"message":"Input is too long for requested model."}}',
|
||||
|
|
@ -1328,6 +1331,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
count_options=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"Message":"Bearer Token has expired"}',
|
||||
|
|
@ -1387,6 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
client_oauth_token=None,
|
||||
count_options=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
|
|
@ -1631,6 +1636,7 @@ class _ScriptedCounter(BaseTokenCounter):
|
|||
output_format: JsonValue = None,
|
||||
context_management: object = None,
|
||||
chat_params: object = None,
|
||||
count_options: object = None,
|
||||
) -> TokenCountResponse | None:
|
||||
self.calls.append((model_to_use, messages, tools, system))
|
||||
self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name"))
|
||||
|
|
@ -4159,3 +4165,242 @@ def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newlin
|
|||
assert line["request_model"] == forged_model
|
||||
assert "\n" not in message
|
||||
assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ')
|
||||
|
||||
|
||||
_COUNTED_OPTION_FIELDS: Final = ("thinking", "tool_choice", "output_config")
|
||||
_DEEPSEEK_COUNT_ROUTE: Final = "https://api.deepseek.com/anthropic/v1/messages/count_tokens"
|
||||
_CUSTOM_READ_FILE_TOOL: Final[dict[str, JsonValue]] = {**_READ_FILE_TOOL, "type": "custom"}
|
||||
|
||||
|
||||
def _messages_inference_fields(
|
||||
config: object, model: str, body: Mapping[str, JsonValue], fields: tuple[str, ...]
|
||||
) -> dict[str, object]:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
inference: Final = config.transform_anthropic_messages_request(
|
||||
model=model,
|
||||
messages=json.loads(json.dumps(body["messages"])),
|
||||
anthropic_messages_optional_request_params={
|
||||
**{key: json.loads(json.dumps(value)) for key, value in body.items() if key not in ("model", "messages")},
|
||||
"max_tokens": 4096,
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
return {field: inference.get(field) for field in fields}
|
||||
|
||||
|
||||
async def _count_messages_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_params: Mapping[str, JsonValue],
|
||||
body: Mapping[str, JsonValue],
|
||||
count_route: str,
|
||||
provider_reply: httpx.Response,
|
||||
) -> tuple[object, respx.Route]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False)
|
||||
monkeypatch.delenv("DEEPSEEK_ANTHROPIC_API_BASE", raising=False)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
Router(model_list=[{"model_name": "counted", "litellm_params": dict(deployment_params)}]),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
route: Final = respx.post(count_route).mock(return_value=provider_reply)
|
||||
response: Final = await anthropic_count_tokens(
|
||||
_json_request({"model": "counted", **body}, "python-httpx/0.28.1"), UserAPIKeyAuth()
|
||||
)
|
||||
return response, route
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "options"),
|
||||
(
|
||||
("claude-sonnet-4-5", {"tool_choice": {"type": "any"}}),
|
||||
("claude-sonnet-4-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}}),
|
||||
("claude-haiku-5-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}, "tool_choice": {"type": "auto"}}),
|
||||
("claude-opus-5-5", {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}),
|
||||
),
|
||||
)
|
||||
async def test_messages_count_tokens_counts_the_thinking_tool_choice_and_effort_messages_inference_sends(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, options: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"messages": [{"role": "user", "content": "Read the README"}],
|
||||
"tools": [_READ_FILE_TOOL],
|
||||
**options,
|
||||
}
|
||||
|
||||
response, route = await _count_messages_body(
|
||||
monkeypatch,
|
||||
{"model": f"anthropic/{model}", "api_key": "sk-ant-test"},
|
||||
body,
|
||||
"https://api.anthropic.com/v1/messages/count_tokens",
|
||||
httpx.Response(200, json={"input_tokens": 120}),
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert response == {"input_tokens": 120, "litellm_count_source": "provider"}
|
||||
assert {field: sent.get(field) for field in _COUNTED_OPTION_FIELDS} == _messages_inference_fields(
|
||||
AnthropicMessagesConfig(), model, body, _COUNTED_OPTION_FIELDS
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_count_tokens_leaves_out_an_option_the_request_drops(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
response, route = await _count_messages_body(
|
||||
monkeypatch,
|
||||
{"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-ant-test"},
|
||||
{
|
||||
"messages": [{"role": "user", "content": "Read the README"}],
|
||||
"tools": [_READ_FILE_TOOL],
|
||||
"tool_choice": {"type": "any"},
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048},
|
||||
"additional_drop_params": ["thinking"],
|
||||
},
|
||||
"https://api.anthropic.com/v1/messages/count_tokens",
|
||||
httpx.Response(200, json={"input_tokens": 120}),
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert response == {"input_tokens": 120, "litellm_count_source": "provider"}
|
||||
assert (sent["tool_choice"], "thinking" in sent) == ({"type": "any"}, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_count_tokens_answers_400_for_a_thinking_budget_messages_inference_rejects(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"messages": [{"role": "user", "content": "Read the README"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": "lots"},
|
||||
}
|
||||
with pytest.raises(BadRequestError) as rejected:
|
||||
_messages_inference_fields(AnthropicMessagesConfig(), "claude-opus-5-5", body, _COUNTED_OPTION_FIELDS)
|
||||
|
||||
response, route = await _count_messages_body(
|
||||
monkeypatch,
|
||||
{"model": "anthropic/claude-opus-5-5", "api_key": "sk-ant-test"},
|
||||
body,
|
||||
"https://api.anthropic.com/v1/messages/count_tokens",
|
||||
httpx.Response(200, json={"input_tokens": 120}),
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert rejected.value.message in json.loads(response.body)["error"]["message"]
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_count_tokens_counts_deepseek_on_its_anthropic_count_route(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig
|
||||
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"messages": [{"role": "user", "content": "Read the README"}],
|
||||
"tools": [_CUSTOM_READ_FILE_TOOL],
|
||||
"tool_choice": {"type": "any"},
|
||||
"thinking": {"type": "enabled"},
|
||||
"output_config": {"effort": "max"},
|
||||
}
|
||||
|
||||
response, route = await _count_messages_body(
|
||||
monkeypatch,
|
||||
{"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test"},
|
||||
body,
|
||||
_DEEPSEEK_COUNT_ROUTE,
|
||||
httpx.Response(200, json={"input_tokens": 91}),
|
||||
)
|
||||
|
||||
sent_request: Final = route.calls.last.request
|
||||
sent: Final = json.loads(sent_request.content)
|
||||
inference: Final = _messages_inference_fields(
|
||||
DeepSeekAnthropicMessagesConfig(), "deepseek-v4-pro", body, ("tool_choice", "output_config", "tools", "thinking")
|
||||
)
|
||||
assert response == {"input_tokens": 91, "litellm_count_source": "provider"}
|
||||
assert sent_request.headers["x-api-key"] == "sk-deepseek-test"
|
||||
assert (sent["tool_choice"], sent.get("output_config"), sent["tools"], sent["thinking"]["type"]) == (
|
||||
inference["tool_choice"],
|
||||
inference["output_config"],
|
||||
inference["tools"],
|
||||
inference["thinking"]["type"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_count_tokens_on_deepseek_estimates_when_the_count_route_is_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, route = await _count_messages_body(
|
||||
monkeypatch,
|
||||
{"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test", "api_base": "https://gateway.example"},
|
||||
{"messages": [{"role": "user", "content": "Read the README"}]},
|
||||
"https://gateway.example/anthropic/v1/messages/count_tokens",
|
||||
httpx.Response(404, json={"error": "not found"}),
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert isinstance(response, dict)
|
||||
assert response["litellm_count_source"] == "local_estimate"
|
||||
assert response["input_tokens"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("reasoning", "expected_options"),
|
||||
(
|
||||
({}, {}),
|
||||
({"reasoning": {"effort": "none"}}, {"thinking": {"type": "disabled"}}),
|
||||
({"reasoning": {"effort": "high"}}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "high"}}),
|
||||
({"reasoning": {"effort": "max"}}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "max"}}),
|
||||
),
|
||||
)
|
||||
async def test_responses_count_on_deepseek_maps_reasoning_effort_the_way_deepseek_chat_inference_does(
|
||||
monkeypatch: pytest.MonkeyPatch, reasoning: Mapping[str, JsonValue], expected_options: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False)
|
||||
monkeypatch.delenv("DEEPSEEK_ANTHROPIC_API_BASE", raising=False)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "deepseek-counted",
|
||||
"litellm_params": {"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.post(_DEEPSEEK_COUNT_ROUTE).mock(return_value=httpx.Response(200, json={"input_tokens": 91}))
|
||||
result: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(model="deepseek-counted", messages=[{"role": "user", "content": "Say hi."}]),
|
||||
call_endpoint=True,
|
||||
responses_request={
|
||||
"model": "deepseek-counted",
|
||||
"input": "Say hi.",
|
||||
"tools": [_WEATHER_TOOL],
|
||||
**reasoning,
|
||||
},
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (result.total_tokens, result.count_source) == (91, "provider")
|
||||
assert {field: sent[field] for field in ("thinking", "output_config") if field in sent} == expected_options
|
||||
assert all("type" not in tool for tool in sent["tools"])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue