From 493cf79031fccb7de6dd8bda0d47f43976d1d838 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:43:29 -0700 Subject: [PATCH] fix(proxy): count thinking, tool_choice, and effort the way inference sends them, and count DeepSeek on its own count route /v1/messages/count_tokens dropped the body's thinking, tool_choice, and output_config, so a Claude count with tool_choice any came out lower than the bill. The route now passes those options to the provider counter, which runs them through the deployment's own /v1/messages transform first, so the count sees what inference sends. A body inference would reject now answers the same 400 before any provider call. DeepSeek had no provider counter and fell back to a local estimate that ignores thinking. It now counts on DeepSeek's Anthropic-compatible count_tokens route with the same base, key, tool sanitizer, and reasoning mapping inference uses, and falls back to the estimate only when that route is missing. --- .../anthropic/count_tokens/token_counter.py | 56 +++- .../anthropic/count_tokens/transformation.py | 70 ++++- .../anthropic/count_tokens/token_counter.py | 56 +++- litellm/llms/base_llm/base_utils.py | 1 + .../count_tokens/bedrock_token_counter.py | 1 + litellm/llms/deepseek/chat/transformation.py | 63 ++--- litellm/llms/deepseek/common_utils.py | 46 ++++ .../deepseek/count_tokens/token_counter.py | 211 +++++++++++++++ .../llms/deepseek/messages/transformation.py | 60 +++-- litellm/llms/gemini/common_utils.py | 1 + .../responses/count_tokens/token_counter.py | 1 + litellm/llms/vertex_ai/common_utils.py | 63 ++--- .../vertex_ai_partner_models/main.py | 33 ++- .../proxy/anthropic_endpoints/endpoints.py | 8 + litellm/proxy/proxy_server.py | 6 + litellm/utils.py | 4 + ...t_anthropic_count_tokens_transformation.py | 121 ++++++++- .../llms/deepseek/count_tokens/__init__.py | 0 .../count_tokens/test_token_counter.py | 213 +++++++++++++++ .../deepseek/test_deepseek_common_utils.py | 14 + .../vertex_ai/test_vertex_ai_common_utils.py | 39 ++- tests/unit/proxy/test_proxy_token_counter.py | 245 ++++++++++++++++++ 22 files changed, 1165 insertions(+), 147 deletions(-) create mode 100644 litellm/llms/deepseek/common_utils.py create mode 100644 litellm/llms/deepseek/count_tokens/token_counter.py create mode 100644 tests/unit/llms/deepseek/count_tokens/__init__.py create mode 100644 tests/unit/llms/deepseek/count_tokens/test_token_counter.py create mode 100644 tests/unit/llms/deepseek/test_deepseek_common_utils.py diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..83dc8eb7659 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -2,7 +2,7 @@ Anthropic Token Counter implementation using the CountTokens API. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Any, Final @@ -11,11 +11,13 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.asyncify import asyncify from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, ChatCountTranslationError, anthropic_count_fields, + messages_count_options, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse @@ -64,12 +66,11 @@ class AnthropicTokenCounter(BaseTokenCounter): Returns: TokenCountResponse with token count, or None if counting fails """ - from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo - if not messages: return None from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig match translated := await anthropic_count_fields( model_to_use, messages, system, tools, messages_format, AnthropicConfig(), chat_params @@ -77,24 +78,51 @@ class AnthropicTokenCounter(BaseTokenCounter): case ChatCountTranslationError(): return translated.response(request_model, model_to_use, "anthropic_api") case AnthropicCountFields(): - return await self.count_tokens( + return await self._count( model_to_use=model_to_use, - messages=[dict(message) for message in translated.messages], - contents=contents, + messages=translated.messages, deployment=deployment, request_model=request_model, - tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None, + tools=translated.tools, system=translated.system, - messages_format="anthropic", - output_format=output_format, - context_management=context_management, count_options=translated.options, ) case None: pass - deployment = deployment or {} - litellm_params: Final = deployment.get("litellm_params", {}) + match options := await asyncify(messages_count_options)( + AnthropicMessagesConfig(), + model_to_use, + messages, + count_options, + (deployment or {}).get("litellm_params", {}), + ): + case ChatCountTranslationError(): + return options.response(request_model, model_to_use, "anthropic_api") + case _: + return await self._count( + model_to_use=model_to_use, + messages=messages, + deployment=deployment, + request_model=request_model, + tools=tools, + system=system, + count_options=options, + ) + + async def _count( + self, + model_to_use: str, + messages: Sequence[Mapping[str, JsonValue]], + deployment: Mapping[str, Any] | None, + request_model: str, + tools: Sequence[Mapping[str, JsonValue]] | None, + system: JsonValue, + count_options: Mapping[str, JsonValue], + ) -> TokenCountResponse | None: + from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo + + litellm_params: Final = (deployment or {}).get("litellm_params", {}) api_base: Final = litellm_params.get("api_base") try: @@ -125,10 +153,10 @@ class AnthropicTokenCounter(BaseTokenCounter): result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, - messages=messages, + messages=[dict(message) for message in messages], auth_header=auth_header, api_base=api_base, - tools=tools, + tools=None if tools is None else [dict(tool) for tool in tools], system=system, optional_params=count_options, ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..3d8bbe9cfc2 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -7,9 +7,11 @@ This module handles the transformation of requests to Anthropic's CountTokens AP import copy from collections.abc import Mapping, Sequence from dataclasses import dataclass +from functools import reduce from itertools import chain from types import MappingProxyType from typing import ( + TYPE_CHECKING, Final, Protocol, TypeAlias, @@ -19,8 +21,9 @@ from typing import ( from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict -from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( anthropic_system_blocks, split_leading_system_run, @@ -29,9 +32,16 @@ from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +if TYPE_CHECKING: + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) +_COUNTED_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) +_TRANSFORMED_REQUEST: Final = TypeAdapter(dict[str, object]) +_DROPPED_PATHS: Final = TypeAdapter(tuple[str, ...]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject) COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config") @@ -325,6 +335,64 @@ def anthropic_fields_for_chat_tool_use( ) +def _max_output_tokens(model: str, custom_llm_provider: str | None) -> int: + from litellm.utils import get_model_info + + try: + max_output_tokens: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider).get( + "max_output_tokens" + ) + except Exception: # noqa: BLE001 # get_model_info raises a plain Exception for a model the cost map lacks + return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS + return ( + max_output_tokens + if isinstance(max_output_tokens, int) and max_output_tokens > 0 + else DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS + ) + + +def without_dropped_paths(options: Mapping[str, JsonValue], additional_drop_params: object) -> Mapping[str, JsonValue]: + return _COUNT_REQUEST.validate_python( + reduce(delete_nested_value, _DROPPED_PATHS.validate_python(additional_drop_params or ()), {**options}) + ) + + +def messages_count_options( + translator: "AnthropicMessagesConfig", + model: str, + messages: Sequence[Mapping[str, JsonValue]], + options: Mapping[str, JsonValue], + litellm_params: Mapping[str, object], +) -> Mapping[str, JsonValue] | ChatCountTranslationError: + """The thinking, tool_choice, and output_config the deployment's /v1/messages transform sends upstream + for these Anthropic Messages body options. The transform needs max_tokens, which a count request lacks, + so it gets the model's max output tokens: a thinking budget's size never changes a count, only whether + thinking stays on.""" + if not options: + return MappingProxyType({}) + from litellm.exceptions import BadRequestError + from litellm.llms.anthropic.common_utils import AnthropicError + + try: + request: Final = _TRANSFORMED_REQUEST.validate_python( + translator.transform_anthropic_messages_request( + model=model, + messages=_COUNTED_MESSAGES.validate_python(messages), + anthropic_messages_optional_request_params={ + **without_dropped_paths(options, litellm_params.get("additional_drop_params")), + "max_tokens": _max_output_tokens(model, translator.custom_llm_provider), + }, + litellm_params=GenericLiteLLMParams.model_validate({"drop_params": litellm_params.get("drop_params")}), + headers={}, + ) + ) + except (AnthropicError, BadRequestError) as e: + return ChatCountTranslationError(message=str(e.message)) + return _COUNT_REQUEST.validate_python( + {name: request[name] for name in COUNT_TOKEN_OPTION_NAMES if request.get(name) is not None} + ) + + async def anthropic_count_fields( model: str, messages: Sequence[Mapping[str, object]], diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index af7cf9d81ce..9019414ef64 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -3,7 +3,7 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API. """ import os -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Any, Final @@ -11,10 +11,12 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.asyncify import asyncify from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, ChatCountTranslationError, anthropic_count_fields, + messages_count_options, ) from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, @@ -78,11 +80,10 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): Returns: TokenCountResponse with token count, or None if counting fails """ - from litellm.llms.anthropic.common_utils import AnthropicError - if not messages: return None + from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig match translated := await anthropic_count_fields( @@ -91,24 +92,51 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): case ChatCountTranslationError(): return translated.response(request_model, model_to_use, "azure_ai_anthropic_api") case AnthropicCountFields(): - return await self.count_tokens( + return await self._count( model_to_use=model_to_use, - messages=[dict(message) for message in translated.messages], - contents=contents, + messages=translated.messages, deployment=deployment, request_model=request_model, - tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None, + tools=translated.tools, system=translated.system, - messages_format="anthropic", - output_format=output_format, - context_management=context_management, count_options=translated.options, ) case None: pass - deployment = deployment or {} - litellm_params: Final = deployment.get("litellm_params", {}) + match options := await asyncify(messages_count_options)( + AzureAnthropicMessagesConfig(), + model_to_use, + messages, + count_options, + (deployment or {}).get("litellm_params", {}), + ): + case ChatCountTranslationError(): + return options.response(request_model, model_to_use, "azure_ai_anthropic_api") + case _: + return await self._count( + model_to_use=model_to_use, + messages=messages, + deployment=deployment, + request_model=request_model, + tools=tools, + system=system, + count_options=options, + ) + + async def _count( + self, + model_to_use: str, + messages: Sequence[Mapping[str, JsonValue]], + deployment: Mapping[str, Any] | None, + request_model: str, + tools: Sequence[Mapping[str, JsonValue]] | None, + system: object | None, + count_options: Mapping[str, JsonValue], + ) -> TokenCountResponse | None: + from litellm.llms.anthropic.common_utils import AnthropicError + + litellm_params: Final = (deployment or {}).get("litellm_params", {}) # Get Azure AI API key from deployment config or environment api_key = litellm_params.get("api_key") @@ -128,11 +156,11 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): try: result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, - messages=messages, + messages=[dict(message) for message in messages], api_key=api_key, api_base=api_base, litellm_params=litellm_params, - tools=tools, + tools=None if tools is None else [dict(tool) for tool in tools], system=system, optional_params=count_options, ) diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ff059190ed5 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -42,6 +42,7 @@ class BaseTokenCounter(ABC): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: pass diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 26bc5904e06..461c9a3431b 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -143,6 +143,7 @@ class BedrockTokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: if not messages: return None diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index ecaa0a2ef16..12f5f2119f0 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -3,9 +3,11 @@ Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completi """ from collections.abc import Coroutine, Mapping, Sequence +from types import MappingProxyType from typing import Final, Literal, cast, overload import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, @@ -20,6 +22,35 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") +def _thinking_type(thinking: object, reasoning_effort: object) -> str | None: + requested: Final = as_str_mapping(thinking) + requested_type: Final = None if requested is None else requested.get("type") + if isinstance(requested_type, str) and requested_type in ("enabled", "disabled"): + return requested_type + if reasoning_effort is None: + return None + return "disabled" if reasoning_effort == "none" else "enabled" + + +def deepseek_reasoning_params(thinking: object, reasoning_effort: object) -> Mapping[str, object]: + """DeepSeek takes `{"type": "enabled"}` or `{"type": "disabled"}` with no budget_tokens. A thinking + param wins. Otherwise `reasoning_effort="none"` turns thinking off and any other effort turns it on, + and a supported effort is sent only while thinking is on. + + Reference: https://api-docs.deepseek.com/guides/thinking_mode + """ + thinking_type: Final = _thinking_type(thinking, reasoning_effort) + if thinking_type is None: + return MappingProxyType({}) + if ( + thinking_type == "enabled" + and isinstance(reasoning_effort, str) + and reasoning_effort in SUPPORTED_REASONING_EFFORTS + ): + return MappingProxyType({"thinking": {"type": thinking_type}, "reasoning_effort": reasoning_effort}) + return MappingProxyType({"thinking": {"type": thinking_type}}) + + class DeepSeekChatConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list: """ @@ -36,40 +67,10 @@ class DeepSeekChatConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict[str, object]: - """ - Map OpenAI params to DeepSeek params. - - Handles `thinking` and `reasoning_effort` parameters for DeepSeek reasoner models. - DeepSeek supports `{"type": "enabled"}` and `{"type": "disabled"}` - no budget_tokens - like Anthropic. `reasoning_effort="none"` is the OpenAI-style way to ask for thinking - off, so it maps to `{"type": "disabled"}`; any other effort keeps thinking on. - - Reference: https://api-docs.deepseek.com/guides/thinking_mode - """ - # Let parent handle standard params first optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) - - # Pop thinking/reasoning_effort from optional_params first (parent may have added them) - # Then re-add only if valid for DeepSeek thinking_value: Final = optional_params.pop("thinking", None) reasoning_effort: Final = optional_params.pop("reasoning_effort", None) - - # Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens - if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"): - optional_params["thinking"] = {"type": thinking_value["type"]} - - # Otherwise fall back to reasoning_effort: "none" disables, anything else enables - elif reasoning_effort is not None: - optional_params["thinking"] = {"type": "disabled" if reasoning_effort == "none" else "enabled"} - - if ( - isinstance(reasoning_effort, str) - and reasoning_effort in SUPPORTED_REASONING_EFFORTS - and optional_params["thinking"] == {"type": "enabled"} - ): - return {**optional_params, "reasoning_effort": reasoning_effort} - - return optional_params + return {**optional_params, **deepseek_reasoning_params(thinking_value, reasoning_effort)} def _fill_reasoning_content(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: """ diff --git a/litellm/llms/deepseek/common_utils.py b/litellm/llms/deepseek/common_utils.py new file mode 100644 index 00000000000..d4a160b839b --- /dev/null +++ b/litellm/llms/deepseek/common_utils.py @@ -0,0 +1,46 @@ +from collections.abc import Mapping, Sequence +from typing import Final + +from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter +from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig, deepseek_anthropic_base +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import LlmProviders + + +class DeepSeekModelInfo(BaseLLMModelInfo): + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + from litellm.utils import get_valid_models + + return get_valid_models(custom_llm_provider=LlmProviders.DEEPSEEK.value) + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return DeepSeekAnthropicMessagesConfig.get_api_key(api_key) + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return deepseek_anthropic_base(api_base) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: matches validate_environment's dict contract shared by every provider override + resolved_api_key: Final = self.get_api_key(api_key) + return {**headers, **({} if resolved_api_key is None else {"x-api-key": resolved_api_key})} + + @staticmethod + def get_base_model(model: str) -> str | None: + return model.removeprefix(f"{LlmProviders.DEEPSEEK.value}/") + + def get_token_counter(self) -> BaseTokenCounter | None: + from litellm.llms.deepseek.count_tokens.token_counter import DeepSeekTokenCounter + + return DeepSeekTokenCounter() diff --git a/litellm/llms/deepseek/count_tokens/token_counter.py b/litellm/llms/deepseek/count_tokens/token_counter.py new file mode 100644 index 00000000000..93d9b3a76a9 --- /dev/null +++ b/litellm/llms/deepseek/count_tokens/token_counter.py @@ -0,0 +1,211 @@ +""" +DeepSeek token counting through the count_tokens route of its Anthropic-compatible API. +""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +from pydantic import JsonValue, TypeAdapter + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler +from litellm.llms.anthropic.count_tokens.transformation import ( + AnthropicCountFields, + ChatCountTranslationError, + anthropic_count_fields, + messages_count_options, +) +from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.llms.deepseek.chat.transformation import deepseek_reasoning_params +from litellm.llms.deepseek.common_utils import DeepSeekModelInfo +from litellm.llms.deepseek.messages.transformation import ( + DeepSeekAnthropicMessagesConfig, + sanitize_tools_for_deepseek, +) +from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse + +DEEPSEEK_TOKENIZER_TYPE: Final = "deepseek_api" +_CHAT_REASONING_PARAMS: Final = ("thinking", "reasoning_effort") +_COUNT_ROUTE_MISSING: Final = frozenset({404, 405}) +_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_OPTIONS: Final = TypeAdapter(dict[str, JsonValue]) + +deepseek_count_tokens_handler: Final = AnthropicCountTokensHandler() + + +def _chat_reasoning_options(chat_params: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + mapped: Final = deepseek_reasoning_params(chat_params.get("thinking"), chat_params.get("reasoning_effort")) + effort: Final = mapped.get("reasoning_effort") + return _OPTIONS.validate_python( + { + **({} if mapped.get("thinking") is None else {"thinking": mapped["thinking"]}), + **({} if effort is None else {"output_config": {"effort": effort}}), + } + ) + + +def _with_chat_reasoning( + options: Mapping[str, JsonValue], reasoning: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + output_config: Final = options.get("output_config") + reasoning_output_config: Final = reasoning.get("output_config") + return _OPTIONS.validate_python( + { + **options, + **reasoning, + **( + {"output_config": {**output_config, **reasoning_output_config}} + if isinstance(output_config, dict) and isinstance(reasoning_output_config, dict) + else {} + ), + } + ) + + +def _litellm_params(deployment: Mapping[str, object] | None) -> Mapping[str, object]: + params: Final = None if deployment is None else as_str_mapping(deployment.get("litellm_params")) + return MappingProxyType({}) if params is None else params + + +def _optional_str(value: object) -> str | None: + return value if isinstance(value, str) else None + + +class DeepSeekTokenCounter(BaseTokenCounter): + """Counts with the request DeepSeek bills: an Anthropic Messages body sent to the Anthropic-compatible + count_tokens route, with thinking and effort mapped the way the deployment's inference maps them.""" + + def __init__(self, handler: AnthropicCountTokensHandler = deepseek_count_tokens_handler) -> None: + self._handler: Final = handler + + def should_use_token_counting_api(self, custom_llm_provider: str | None = None) -> bool: + return custom_llm_provider == LlmProviders.DEEPSEEK.value + + async def count_tokens( + self, + model_to_use: str, + messages: Sequence[Mapping[str, JsonValue]] | None, + contents: Sequence[Mapping[str, object]] | None, + deployment: Mapping[str, object] | None = None, + request_model: str = "", + tools: Sequence[Mapping[str, JsonValue]] | None = None, + system: JsonValue = None, + messages_format: TokenCountMessagesFormat = "anthropic", + output_format: JsonValue = None, + context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), + ) -> TokenCountResponse | None: + if not messages: + return None + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + litellm_params: Final = _litellm_params(deployment) + match translated := await anthropic_count_fields( + model_to_use, + messages, + system, + tools, + messages_format, + AnthropicConfig(), + {name: value for name, value in chat_params.items() if name not in _CHAT_REASONING_PARAMS}, + ): + case ChatCountTranslationError(): + return translated.response(request_model, model_to_use, DEEPSEEK_TOKENIZER_TYPE) + case AnthropicCountFields(): + return await self._count( + model_to_use=model_to_use, + messages=translated.messages, + litellm_params=litellm_params, + request_model=request_model, + tools=translated.tools, + system=translated.system, + count_options=_with_chat_reasoning(translated.options, _chat_reasoning_options(chat_params)), + ) + case None: + pass + + if messages_format == "openai": + return await self._count( + model_to_use=model_to_use, + messages=messages, + litellm_params=litellm_params, + request_model=request_model, + tools=tools, + system=system, + count_options=_chat_reasoning_options(chat_params), + ) + + match options := await asyncify(messages_count_options)( + DeepSeekAnthropicMessagesConfig(), model_to_use, messages, count_options, litellm_params + ): + case ChatCountTranslationError(): + return options.response(request_model, model_to_use, DEEPSEEK_TOKENIZER_TYPE) + case _: + return await self._count( + model_to_use=model_to_use, + messages=messages, + litellm_params=litellm_params, + request_model=request_model, + tools=tools, + system=system, + count_options=options, + ) + + async def _count( + self, + model_to_use: str, + messages: Sequence[Mapping[str, JsonValue]], + litellm_params: Mapping[str, object], + request_model: str, + tools: Sequence[Mapping[str, JsonValue]] | None, + system: JsonValue, + count_options: Mapping[str, JsonValue], + ) -> TokenCountResponse | None: + model_info: Final = DeepSeekModelInfo() + try: + result: Final = await self._handler.handle_count_tokens_request( + model=model_to_use, + messages=[dict(message) for message in messages], + auth_header=model_info.validate_environment( + headers={}, + model=model_to_use, + messages=(), + optional_params={}, + litellm_params=litellm_params, + api_key=_optional_str(litellm_params.get("api_key")), + ), + api_base=model_info.get_api_base(_optional_str(litellm_params.get("api_base"))), + tools=None if tools is None else _TOOLS.validate_python(sanitize_tools_for_deepseek(tools)), + system=system, + optional_params=count_options, + ) + except AnthropicError as e: + if e.status_code in _COUNT_ROUTE_MISSING: + verbose_logger.debug( + "DeepSeek count_tokens route answered %s, so the count is estimated", e.status_code + ) + return None + verbose_logger.warning("DeepSeek CountTokens error: status=%s, message=%s", e.status_code, e.message) + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type=DEEPSEEK_TOKENIZER_TYPE, + error=True, + error_message=e.message, + status_code=e.status_code, + ) + input_tokens: Final = result.get("input_tokens") + return TokenCountResponse( + total_tokens=input_tokens if isinstance(input_tokens, int) else 0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type=DEEPSEEK_TOKENIZER_TYPE, + original_response=result, + ) diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 3b2076b0085..9da594ee2ff 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -16,6 +16,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams _TOOL_LIST: Final = TypeAdapter(list[object]) +_REQUEST_FIELDS: Final = TypeAdapter(dict[str, object]) _TOOL_FIELDS: Final = TypeAdapter(dict[object, object]) @@ -34,6 +35,20 @@ def _sanitized_tool(tool: object) -> object: return tool if stripped is None else stripped +def sanitize_tools_for_deepseek(tools: Sequence[object]) -> Sequence[object]: + return [_sanitized_tool(tool) for tool in tools] + + +def deepseek_anthropic_base(api_base: str | None) -> str: + configured: Final = DeepSeekAnthropicMessagesConfig.get_api_base(api_base=api_base).rstrip("/") + if configured.endswith("/v1/messages") and "/anthropic/" in configured: + return configured.removesuffix("/v1/messages") + stripped: Final = configured.removesuffix("/v1/messages").removesuffix("/v1").removesuffix("/beta") + if stripped.endswith("/anthropic") or "/anthropic/" in stripped: + return stripped + return join_api_base(stripped, "anthropic") + + class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): """ DeepSeek exposes an Anthropic-compatible Messages API at @@ -102,25 +117,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - base_url = self.get_api_base(api_base=api_base).rstrip("/") - - if base_url.endswith("/v1/messages") and "/anthropic/" in base_url: - return base_url - base_url = base_url.removesuffix("/v1/messages") - base_url = base_url.removesuffix("/v1") - base_url = base_url.removesuffix("/beta") - - if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url: - base_url = join_api_base(base_url, "anthropic") - - return join_api_base(base_url, "v1/messages") - - @staticmethod - def _sanitize_tools_for_deepseek(tools: object) -> object: - if not isinstance(tools, list): - return tools - - return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)] + return join_api_base(deepseek_anthropic_base(api_base), "v1/messages") def transform_anthropic_messages_request( self, @@ -130,13 +127,20 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> dict: - anthropic_messages_request: Final = super().transform_anthropic_messages_request( - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, + anthropic_messages_request: Final = _REQUEST_FIELDS.validate_python( + super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) ) - if "tools" in anthropic_messages_request: - anthropic_messages_request["tools"] = self._sanitize_tools_for_deepseek(anthropic_messages_request["tools"]) - return anthropic_messages_request + tools: Final = anthropic_messages_request.get("tools") + if not isinstance(tools, list): + return anthropic_messages_request + sanitized: Final = { + **anthropic_messages_request, + "tools": sanitize_tools_for_deepseek(_TOOL_LIST.validate_python(tools)), + } + return sanitized diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 88c6494e3f7..da63abb947e 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -534,6 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: import copy diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..4064a435f3e 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ Count tokens using OpenAI's Responses API /input_tokens endpoint. diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 0fa80cb2cad..aade68dba57 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -7,7 +7,7 @@ from enum import Enum from functools import lru_cache from itertools import accumulate, chain, repeat, takewhile from types import MappingProxyType -from typing import Any, Final, Literal, TypeVar, cast, get_type_hints +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast, get_type_hints import httpx from pydantic import JsonValue, TypeAdapter, ValidationError @@ -33,6 +33,9 @@ from litellm.types.llms.vertex_ai import ( from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse from litellm.utils import supports_response_schema, supports_system_messages +if TYPE_CHECKING: + from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountFields + GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType( {"fps": "fps", "start_offset": "startOffset", "end_offset": "endOffset"} ) @@ -1395,21 +1398,17 @@ class VertexAITokenCounter(BaseTokenCounter): return custom_llm_provider == LlmProviders.VERTEX_AI.value - async def _count_translated_claude_request( + async def _translated_claude_request( self, model_to_use: str, messages: Sequence[Mapping[str, object]] | None, - deployment: Mapping[str, object], request_model: str, tools: Sequence[Mapping[str, object]] | None, system: object | None, messages_format: TokenCountMessagesFormat, - output_format: JsonValue, - context_management: CountContextManagement | None, chat_params: Mapping[str, JsonValue], - ) -> TokenCountResponse | None: + ) -> "TokenCountResponse | AnthropicCountFields | None": from litellm.llms.anthropic.count_tokens.transformation import ( - AnthropicCountFields, ChatCountTranslationError, anthropic_count_fields, ) @@ -1419,27 +1418,12 @@ class VertexAITokenCounter(BaseTokenCounter): return None from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig - match translated := await anthropic_count_fields( + translated: Final = await anthropic_count_fields( model_to_use, messages or [], system, tools, messages_format, VertexAIAnthropicConfig(), chat_params - ): - case ChatCountTranslationError(): - return translated.response(request_model, model_to_use, "vertex_ai_partner_models") - case AnthropicCountFields(): - return await self.count_tokens( - model_to_use=model_to_use, - messages=[dict(message) for message in translated.messages], - contents=None, - deployment=dict(deployment), - request_model=request_model, - tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None, - system=translated.system, - messages_format="anthropic", - output_format=output_format, - context_management=context_management, - count_options=translated.options, - ) - case None: - return None + ) + if isinstance(translated, ChatCountTranslationError): + return translated.response(request_model, model_to_use, "vertex_ai_partner_models") + return translated async def count_tokens( self, @@ -1493,20 +1477,17 @@ class VertexAITokenCounter(BaseTokenCounter): # Check if this is a partner model (Claude, Mistral, etc.) if VertexAIPartnerModels.is_vertex_partner_model(model_to_use): - translated_count: Final = await self._count_translated_claude_request( + translated: Final = await self._translated_claude_request( model_to_use=model_to_use, messages=messages, - deployment=deployment, request_model=request_model, tools=tools, system=system, messages_format=messages_format, - output_format=output_format, - context_management=context_management, chat_params=chat_params, ) - if translated_count is not None: - return translated_count + if isinstance(translated, TokenCountResponse): + return translated # Use partner models token counter partner_models_handler: Final = VertexAIPartnerModels() @@ -1531,14 +1512,22 @@ class VertexAITokenCounter(BaseTokenCounter): try: result = await partner_models_handler.count_tokens( model=model_to_use, - messages=messages or [], + messages=( + (messages or []) if translated is None else [dict(message) for message in translated.messages] + ), litellm_params=partner_litellm_params, vertex_project=vertex_project, vertex_location=vertex_location, vertex_credentials=vertex_credentials, - system=system, - tools=tools, - count_options=count_options, + system=system if translated is None else translated.system, + tools=( + tools + if translated is None + else None + if translated.tools is None + else [dict(tool) for tool in translated.tools] + ), + count_options=MappingProxyType({}) if translated is None else translated.options, ) except (PartnerVertexAIError, httpx.HTTPStatusError) as e: status_code: Final = e.response.status_code diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index bb200fd542d..17aa353239a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -1,6 +1,6 @@ # What is this? ## API Handler for calling Vertex AI Partner Models -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import Final @@ -259,13 +259,13 @@ class VertexAIPartnerModels(VertexBase): async def count_tokens( self, model: str, - messages: list, - litellm_params: dict, - vertex_project=None, - vertex_location=None, - vertex_credentials=None, + messages: Sequence[Mapping[str, object]], + litellm_params: Mapping[str, object], + vertex_project: object = None, + vertex_location: object = None, + vertex_credentials: object = None, system: object | None = None, - tools: list[dict[str, object]] | None = None, + tools: Sequence[Mapping[str, object]] | None = None, count_options: Mapping[str, object] = MappingProxyType({}), ): """ @@ -307,13 +307,18 @@ class VertexAIPartnerModels(VertexBase): } # Prepare litellm_params with credentials - _litellm_params: Final = litellm_params.copy() - if vertex_project: - _litellm_params["vertex_project"] = vertex_project - if vertex_location: - _litellm_params["vertex_location"] = vertex_location - if vertex_credentials: - _litellm_params["vertex_credentials"] = vertex_credentials + _litellm_params: Final = { + **litellm_params, + **{ + key: value + for key, value in ( + ("vertex_project", vertex_project), + ("vertex_location", vertex_location), + ("vertex_credentials", vertex_credentials), + ) + if value + }, + } # Call the token counter token_counter: Final = VertexAIPartnerModelsTokenCounter() diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..f6ce426c950 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -19,6 +19,7 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, without_dropped_paths from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) @@ -342,6 +343,7 @@ class _CountTokensBodyOptions(TypedDict): output_format: ReadOnly[JsonValue] context_management: ReadOnly[JsonValue] request_litellm_params: ReadOnly[Mapping[str, JsonValue]] + count_options: ReadOnly[Mapping[str, JsonValue]] def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: bool) -> _CountTokensBodyOptions: @@ -352,6 +354,12 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: "request_litellm_params": MappingProxyType( {"drop_params": True, **sent_params} if auto_drop_params else sent_params ), + "count_options": MappingProxyType( + without_dropped_paths( + {name: data[name] for name in COUNT_TOKEN_OPTION_NAMES if name in data}, + data.get("additional_drop_params"), + ) + ), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5e37ff1ba65..9e215ca1dc2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14814,6 +14814,7 @@ async def _try_provider_token_count( output_format: JsonValue = None, context_management: CountContextManagement | None = None, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> Union["TokenCountResponse", ProviderCountFailure, None]: """Attempt provider-specific token counting. Returns None to fall through to local counting.""" if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): @@ -14841,6 +14842,7 @@ async def _try_provider_token_count( output_format=output_format, context_management=context_management, chat_params=chat_params, + count_options=count_options, ) except httpx.HTTPStatusError as e: return ProviderCountFailure( @@ -15320,6 +15322,7 @@ async def count_request_tokens( responses_request: Mapping[str, object] | None = None, litellm_call_id: str | None = None, client_oauth_token: str | None = None, + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse: call_id: Final = resolve_litellm_call_id(litellm_call_id) if request.prompt is None and request.messages is None and request.contents is None: @@ -15339,6 +15342,7 @@ async def count_request_tokens( request_litellm_params=request_litellm_params, responses_request=responses_request, client_oauth_token=client_oauth_token, + count_options=count_options, ) return await logged_token_count( _with_anthropic_content_errors(count) if messages_format == "anthropic" else count, @@ -15381,6 +15385,7 @@ async def _count_request_tokens( request_litellm_params: Mapping[str, JsonValue], responses_request: Mapping[str, object] | None, client_oauth_token: str | None, + count_options: Mapping[str, JsonValue], ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15488,6 +15493,7 @@ async def _count_request_tokens( output_format=output_format, context_management=_count_context_management(context_management, request.model, request_litellm_params), chat_params=MappingProxyType({}) if chat_count is None else chat_count.params, + count_options=count_options, ), model_to_use=model_to_use, request_model=request.model, diff --git a/litellm/utils.py b/litellm/utils.py index 6a9beeec916..f16085b03b2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10142,6 +10142,10 @@ class ProviderConfigManager: from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo return AzureFoundryModelInfo(model=model) + elif LlmProviders.DEEPSEEK == provider: + from litellm.llms.deepseek.common_utils import DeepSeekModelInfo + + return DeepSeekModelInfo() return None @staticmethod diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 89352ce8702..2817ef38871 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -16,9 +16,12 @@ import litellm from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter +from litellm.exceptions import BadRequestError from litellm.llms.anthropic.count_tokens.transformation import ( + COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig, ) +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, ) @@ -26,6 +29,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) +from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -482,6 +487,8 @@ async def _counted_body( tools: list[dict[str, JsonValue]] | None, messages_format: TokenCountMessagesFormat, chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), + model: str = "claude-opus-5-5", ) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]: sent: Final[SimpleQueue[bytes]] = SimpleQueue() @@ -492,14 +499,15 @@ async def _counted_body( with respx.mock: respx.post(count_endpoint).mock(side_effect=count) result: Final = await counter_type().count_tokens( - model_to_use="claude-opus-5-5", + model_to_use=model, messages=messages, contents=None, deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, - request_model="claude-opus-5-5", + request_model=model, tools=tools, messages_format=messages_format, chat_params=chat_params, + count_options=count_options, ) return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize())) @@ -842,3 +850,112 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat inference["tool_choice"], inference["tools"], ) + + +_MESSAGES_CONFIGS: Final = MappingProxyType( + {AnthropicTokenCounter: AnthropicMessagesConfig, AzureAIAnthropicTokenCounter: AzureAnthropicMessagesConfig} +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + ("model", "options"), + ( + ("claude-sonnet-4-5", {"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}), + ("claude-haiku-5-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}}), + ("claude-opus-5-5", {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}), + ("claude-unlisted-model", {"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}), + ), +) +async def test_counter_sends_the_thinking_tool_choice_and_effort_messages_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + model: str, + options: Mapping[str, JsonValue], +) -> None: + messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}] + inference: Final = _MESSAGES_CONFIGS[counter_type]().transform_anthropic_messages_request( + model=model, + messages=deepcopy(messages), + anthropic_messages_optional_request_params={**deepcopy(dict(options)), "max_tokens": 4096}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result, bodies = await _counted_body( + counter_type, count_endpoint, messages, None, "anthropic", count_options=options, model=model + ) + + (body,) = bodies + assert result is not None + assert result.total_tokens == 31 + assert {name: body.get(name) for name in COUNT_TOKEN_OPTION_NAMES} == { + name: inference.get(name) for name in COUNT_TOKEN_OPTION_NAMES + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_leaves_out_an_option_the_deployment_drops( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + sent: Final[SimpleQueue[bytes]] = SimpleQueue() + + def count(request: httpx.Request) -> httpx.Response: + sent.put(request.content) + return httpx.Response(200, json={"input_tokens": 31}) + + with respx.mock: + respx.post(count_endpoint).mock(side_effect=count) + await counter_type().count_tokens( + model_to_use="claude-sonnet-4-5", + messages=[{"role": "user", "content": "Weather in SF?"}], + contents=None, + deployment={ + "litellm_params": { + "api_key": "sk-ant-api03-test-key", + "api_base": "https://gateway.example", + "additional_drop_params": ["tool_choice"], + } + }, + request_model="claude-sonnet-4-5", + messages_format="anthropic", + count_options={"tool_choice": {"type": "any"}, "thinking": {"type": "enabled", "budget_tokens": 2048}}, + ) + + body: Final = TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) + assert ("tool_choice" in body, body["thinking"]) == (False, {"type": "enabled", "budget_tokens": 2048}) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_rejects_a_thinking_budget_messages_inference_rejects_without_calling_anthropic( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + options: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": "lots"}} + with pytest.raises(BadRequestError) as rejected: + _MESSAGES_CONFIGS[counter_type]().transform_anthropic_messages_request( + model="claude-opus-5-5", + messages=[{"role": "user", "content": "Weather in SF?"}], + anthropic_messages_optional_request_params={**deepcopy(options), "max_tokens": 4096}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result, bodies = await _counted_body( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + None, + "anthropic", + count_options=options, + ) + + assert bodies == () + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert rejected.value.message in (result.error_message or "") diff --git a/tests/unit/llms/deepseek/count_tokens/__init__.py b/tests/unit/llms/deepseek/count_tokens/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/deepseek/count_tokens/test_token_counter.py b/tests/unit/llms/deepseek/count_tokens/test_token_counter.py new file mode 100644 index 00000000000..55ed3b78d95 --- /dev/null +++ b/tests/unit/llms/deepseek/count_tokens/test_token_counter.py @@ -0,0 +1,213 @@ +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import respx +from pydantic import JsonValue + +import litellm +from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.deepseek.count_tokens.token_counter import DeepSeekTokenCounter +from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse + +_DEFAULT_COUNT_ROUTE: Final = "https://api.deepseek.com/anthropic/v1/messages/count_tokens" +_WEATHER_IN_SF: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}] +_GET_WEATHER: Final[dict[str, JsonValue]] = { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, +} + + +@pytest.fixture(autouse=True) +def _deepseek_env_unset(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("DEEPSEEK_API_BASE", "DEEPSEEK_ANTHROPIC_API_BASE", "DEEPSEEK_API_KEY"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +async def _count( + reply: httpx.Response, + count_route: str = _DEFAULT_COUNT_ROUTE, + litellm_params: Mapping[str, JsonValue] = MappingProxyType({"api_key": "sk-deepseek-test"}), + messages_format: TokenCountMessagesFormat = "anthropic", + tools: list[dict[str, JsonValue]] | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), + messages: list[dict[str, JsonValue]] = _WEATHER_IN_SF, +) -> tuple[TokenCountResponse | None, respx.Route]: + with respx.mock: + route: Final = respx.post(count_route).mock(return_value=reply) + result: Final = await DeepSeekTokenCounter().count_tokens( + model_to_use="deepseek-v4-pro", + messages=messages, + contents=None, + deployment={"litellm_params": dict(litellm_params)}, + request_model="deepseek-pro", + tools=tools, + messages_format=messages_format, + chat_params=chat_params, + count_options=count_options, + ) + return result, route + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "api_base", + ( + None, + "https://api.deepseek.com", + "https://api.deepseek.com/beta", + "https://api.deepseek.com/anthropic/v1/messages", + "https://gateway.example/deepseek", + ), +) +async def test_count_posts_under_the_base_messages_inference_posts_to(api_base: str | None) -> None: + messages_url: Final = DeepSeekAnthropicMessagesConfig().get_complete_url( + api_base=api_base, api_key=None, model="deepseek-v4-pro", optional_params={}, litellm_params={} + ) + + result, route = await _count( + httpx.Response(200, json={"input_tokens": 91}), + count_route=f"{messages_url}/count_tokens", + litellm_params={"api_key": "sk-deepseek-test", **({} if api_base is None else {"api_base": api_base})}, + ) + + assert route.calls.last.request.headers["x-api-key"] == "sk-deepseek-test" + assert result is not None + assert (result.total_tokens, result.error) == (91, False) + + +@pytest.mark.asyncio +async def test_count_without_a_key_sends_no_api_key_header() -> None: + _, route = await _count(httpx.Response(200, json={"input_tokens": 91}), litellm_params={}) + + assert "x-api-key" not in route.calls.last.request.headers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("messages_format", ("anthropic", "openai")) +async def test_count_sends_tools_without_the_custom_type_deepseek_refuses( + messages_format: TokenCountMessagesFormat, +) -> None: + tool: Final[dict[str, JsonValue]] = ( + {**_GET_WEATHER, "type": "custom"} + if messages_format == "anthropic" + else { + "type": "function", + "function": {"name": "get_weather", "parameters": _GET_WEATHER["input_schema"]}, + } + ) + + _, route = await _count( + httpx.Response(200, json={"input_tokens": 91}), messages_format=messages_format, tools=[tool] + ) + + assert json.loads(route.calls.last.request.content)["tools"] == [_GET_WEATHER] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("chat_params", "expected_options"), + ( + ({}, {}), + ({"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}), + ({"reasoning_effort": "max"}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "max"}}), + ({"reasoning_effort": "unlisted"}, {"thinking": {"type": "enabled"}}), + ({"thinking": {"type": "enabled", "budget_tokens": 1024}}, {"thinking": {"type": "enabled"}}), + ), +) +async def test_chat_count_maps_thinking_and_effort_the_way_deepseek_chat_inference_does( + chat_params: Mapping[str, JsonValue], expected_options: Mapping[str, JsonValue] +) -> None: + _, route = await _count( + httpx.Response(200, json={"input_tokens": 91}), messages_format="openai", chat_params=chat_params + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert {name: sent[name] for name in ("thinking", "output_config") if name in sent} == expected_options + + +@pytest.mark.asyncio +async def test_messages_count_sends_the_thinking_messages_inference_sends() -> None: + _, route = await _count( + httpx.Response(200, json={"input_tokens": 91}), + count_options={"thinking": {"type": "disabled"}, "tool_choice": {"type": "any"}}, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["thinking"], sent["tool_choice"]) == ({"type": "disabled"}, {"type": "any"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", (404, 405)) +async def test_a_missing_count_route_leaves_the_count_to_the_estimate(status_code: int) -> None: + result, route = await _count(httpx.Response(status_code, text="no such route")) + + assert route.called + assert result is None + + +@pytest.mark.asyncio +async def test_a_rejected_count_answers_with_deepseeks_status() -> None: + result, _ = await _count(httpx.Response(401, json={"error": {"message": "Authentication Fails"}})) + + assert result is not None + assert (result.error, result.status_code, result.total_tokens) == (True, 401, 0) + assert "Authentication Fails" in (result.error_message or "") + + +@pytest.mark.asyncio +async def test_count_without_messages_leaves_the_count_to_the_estimate() -> None: + result, route = await _count(httpx.Response(200, json={"input_tokens": 91}), messages=[]) + + assert (result, route.called) == (None, False) + + +@pytest.mark.asyncio +async def test_messages_count_rejects_an_effort_messages_inference_rejects_without_calling_deepseek() -> None: + options: Final[dict[str, JsonValue]] = {"output_config": {"effort": "unlisted"}} + with pytest.raises(AnthropicError) as rejected: + DeepSeekAnthropicMessagesConfig().transform_anthropic_messages_request( + model="deepseek-v4-pro", + messages=[dict(message) for message in _WEATHER_IN_SF], + anthropic_messages_optional_request_params={**options, "max_tokens": 4096}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result, route = await _count(httpx.Response(200, json={"input_tokens": 91}), count_options=options) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + assert (rejected.value.status_code, str(rejected.value.message)) == (400, result.error_message) + + +@pytest.mark.asyncio +async def test_chat_count_rejects_a_tool_call_chat_inference_rejects_without_calling_deepseek() -> None: + messages: Final[list[dict[str, JsonValue]]] = [ + *_WEATHER_IN_SF, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ] + + result, route = await _count( + httpx.Response(200, json={"input_tokens": 91}), messages_format="openai", messages=messages + ) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) diff --git a/tests/unit/llms/deepseek/test_deepseek_common_utils.py b/tests/unit/llms/deepseek/test_deepseek_common_utils.py new file mode 100644 index 00000000000..31e86fef5c9 --- /dev/null +++ b/tests/unit/llms/deepseek/test_deepseek_common_utils.py @@ -0,0 +1,14 @@ +import pytest + +import litellm + + +def test_deepseek_model_listing_with_a_provider_check_lists_the_cost_map_models_offline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek-test") + + listed = litellm.get_valid_models(custom_llm_provider="deepseek", check_provider_endpoint=True) + + assert listed + assert sorted(listed) == sorted(litellm.models_by_provider["deepseek"]) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 152ab2ffc13..17f702ecdca 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -31,7 +31,7 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig -from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse @pytest.mark.asyncio @@ -1839,9 +1839,9 @@ _PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = { } -async def _partner_counted_bodies( - model: str, chat_params: dict[str, JsonValue] | None = None -) -> tuple[dict[str, object], ...]: +async def _partner_count( + model: str, messages: list[dict[str, object]], chat_params: dict[str, JsonValue] | None = None +) -> tuple[TokenCountResponse | None, tuple[dict[str, object], ...]]: from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler class FakeResponse: @@ -1870,16 +1870,43 @@ async def _partner_counted_bodies( ): result: Final = await VertexAITokenCounter().count_tokens( model_to_use=model, - messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS), + messages=copy.deepcopy(messages), contents=None, deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, tools=[_PARTNER_RESPONSES_TOOL], chat_params=chat_params or {}, ) + return result, fake_http_client.posted_bodies + +async def _partner_counted_bodies( + model: str, chat_params: dict[str, JsonValue] | None = None +) -> tuple[dict[str, object], ...]: + result, bodies = await _partner_count(model, _PARTNER_CHAT_TOOL_TURNS, chat_params) assert result is not None assert result.total_tokens == 37 - return fake_http_client.posted_bodies + return bodies + + +@pytest.mark.asyncio +async def test_vertex_claude_count_rejects_a_tool_call_chat_inference_rejects_without_calling_vertex() -> None: + unparseable_arguments: Final = [ + _PARTNER_CHAT_TOOL_TURNS[0], + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}} + ], + }, + _PARTNER_CHAT_TOOL_TURNS[2], + ] + + result, bodies = await _partner_count("claude-opus-5-5", unparseable_arguments) + + assert bodies == () + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index b21730c64f6..56c9a095dc6 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -367,6 +367,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + count_options=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -449,6 +450,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + count_options=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -1269,6 +1271,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + count_options=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1328,6 +1331,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + count_options=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1387,6 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): request_litellm_params=None, litellm_call_id=None, client_oauth_token=None, + count_options=None, ): raise ProxyException( message="Rate limit exceeded", @@ -1631,6 +1636,7 @@ class _ScriptedCounter(BaseTokenCounter): output_format: JsonValue = None, context_management: object = None, chat_params: object = None, + count_options: object = None, ) -> TokenCountResponse | None: self.calls.append((model_to_use, messages, tools, system)) self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name")) @@ -4159,3 +4165,242 @@ def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newlin assert line["request_model"] == forged_model assert "\n" not in message assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ') + + +_COUNTED_OPTION_FIELDS: Final = ("thinking", "tool_choice", "output_config") +_DEEPSEEK_COUNT_ROUTE: Final = "https://api.deepseek.com/anthropic/v1/messages/count_tokens" +_CUSTOM_READ_FILE_TOOL: Final[dict[str, JsonValue]] = {**_READ_FILE_TOOL, "type": "custom"} + + +def _messages_inference_fields( + config: object, model: str, body: Mapping[str, JsonValue], fields: tuple[str, ...] +) -> dict[str, object]: + from litellm.types.router import GenericLiteLLMParams + + inference: Final = config.transform_anthropic_messages_request( + model=model, + messages=json.loads(json.dumps(body["messages"])), + anthropic_messages_optional_request_params={ + **{key: json.loads(json.dumps(value)) for key, value in body.items() if key not in ("model", "messages")}, + "max_tokens": 4096, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + return {field: inference.get(field) for field in fields} + + +async def _count_messages_body( + monkeypatch: pytest.MonkeyPatch, + deployment_params: Mapping[str, JsonValue], + body: Mapping[str, JsonValue], + count_route: str, + provider_reply: httpx.Response, +) -> tuple[object, respx.Route]: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + monkeypatch.delenv("DEEPSEEK_ANTHROPIC_API_BASE", raising=False) + monkeypatch.setattr( + proxy_server, + "llm_router", + Router(model_list=[{"model_name": "counted", "litellm_params": dict(deployment_params)}]), + ) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(count_route).mock(return_value=provider_reply) + response: Final = await anthropic_count_tokens( + _json_request({"model": "counted", **body}, "python-httpx/0.28.1"), UserAPIKeyAuth() + ) + return response, route + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "options"), + ( + ("claude-sonnet-4-5", {"tool_choice": {"type": "any"}}), + ("claude-sonnet-4-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}}), + ("claude-haiku-5-5", {"thinking": {"type": "enabled", "budget_tokens": 2048}, "tool_choice": {"type": "auto"}}), + ("claude-opus-5-5", {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}), + ), +) +async def test_messages_count_tokens_counts_the_thinking_tool_choice_and_effort_messages_inference_sends( + monkeypatch: pytest.MonkeyPatch, model: str, options: Mapping[str, JsonValue] +) -> None: + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + + body: Final[dict[str, JsonValue]] = { + "messages": [{"role": "user", "content": "Read the README"}], + "tools": [_READ_FILE_TOOL], + **options, + } + + response, route = await _count_messages_body( + monkeypatch, + {"model": f"anthropic/{model}", "api_key": "sk-ant-test"}, + body, + "https://api.anthropic.com/v1/messages/count_tokens", + httpx.Response(200, json={"input_tokens": 120}), + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert response == {"input_tokens": 120, "litellm_count_source": "provider"} + assert {field: sent.get(field) for field in _COUNTED_OPTION_FIELDS} == _messages_inference_fields( + AnthropicMessagesConfig(), model, body, _COUNTED_OPTION_FIELDS + ) + + +@pytest.mark.asyncio +async def test_messages_count_tokens_leaves_out_an_option_the_request_drops(monkeypatch: pytest.MonkeyPatch) -> None: + response, route = await _count_messages_body( + monkeypatch, + {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-ant-test"}, + { + "messages": [{"role": "user", "content": "Read the README"}], + "tools": [_READ_FILE_TOOL], + "tool_choice": {"type": "any"}, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "additional_drop_params": ["thinking"], + }, + "https://api.anthropic.com/v1/messages/count_tokens", + httpx.Response(200, json={"input_tokens": 120}), + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert response == {"input_tokens": 120, "litellm_count_source": "provider"} + assert (sent["tool_choice"], "thinking" in sent) == ({"type": "any"}, False) + + +@pytest.mark.asyncio +async def test_messages_count_tokens_answers_400_for_a_thinking_budget_messages_inference_rejects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.exceptions import BadRequestError + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + + body: Final[dict[str, JsonValue]] = { + "messages": [{"role": "user", "content": "Read the README"}], + "thinking": {"type": "enabled", "budget_tokens": "lots"}, + } + with pytest.raises(BadRequestError) as rejected: + _messages_inference_fields(AnthropicMessagesConfig(), "claude-opus-5-5", body, _COUNTED_OPTION_FIELDS) + + response, route = await _count_messages_body( + monkeypatch, + {"model": "anthropic/claude-opus-5-5", "api_key": "sk-ant-test"}, + body, + "https://api.anthropic.com/v1/messages/count_tokens", + httpx.Response(200, json={"input_tokens": 120}), + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert rejected.value.message in json.loads(response.body)["error"]["message"] + assert not route.called + + +@pytest.mark.asyncio +async def test_messages_count_tokens_counts_deepseek_on_its_anthropic_count_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig + + body: Final[dict[str, JsonValue]] = { + "messages": [{"role": "user", "content": "Read the README"}], + "tools": [_CUSTOM_READ_FILE_TOOL], + "tool_choice": {"type": "any"}, + "thinking": {"type": "enabled"}, + "output_config": {"effort": "max"}, + } + + response, route = await _count_messages_body( + monkeypatch, + {"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test"}, + body, + _DEEPSEEK_COUNT_ROUTE, + httpx.Response(200, json={"input_tokens": 91}), + ) + + sent_request: Final = route.calls.last.request + sent: Final = json.loads(sent_request.content) + inference: Final = _messages_inference_fields( + DeepSeekAnthropicMessagesConfig(), "deepseek-v4-pro", body, ("tool_choice", "output_config", "tools", "thinking") + ) + assert response == {"input_tokens": 91, "litellm_count_source": "provider"} + assert sent_request.headers["x-api-key"] == "sk-deepseek-test" + assert (sent["tool_choice"], sent.get("output_config"), sent["tools"], sent["thinking"]["type"]) == ( + inference["tool_choice"], + inference["output_config"], + inference["tools"], + inference["thinking"]["type"], + ) + + +@pytest.mark.asyncio +async def test_messages_count_tokens_on_deepseek_estimates_when_the_count_route_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, route = await _count_messages_body( + monkeypatch, + {"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test", "api_base": "https://gateway.example"}, + {"messages": [{"role": "user", "content": "Read the README"}]}, + "https://gateway.example/anthropic/v1/messages/count_tokens", + httpx.Response(404, json={"error": "not found"}), + ) + + assert route.called + assert isinstance(response, dict) + assert response["litellm_count_source"] == "local_estimate" + assert response["input_tokens"] > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reasoning", "expected_options"), + ( + ({}, {}), + ({"reasoning": {"effort": "none"}}, {"thinking": {"type": "disabled"}}), + ({"reasoning": {"effort": "high"}}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "high"}}), + ({"reasoning": {"effort": "max"}}, {"thinking": {"type": "enabled"}, "output_config": {"effort": "max"}}), + ), +) +async def test_responses_count_on_deepseek_maps_reasoning_effort_the_way_deepseek_chat_inference_does( + monkeypatch: pytest.MonkeyPatch, reasoning: Mapping[str, JsonValue], expected_options: Mapping[str, JsonValue] +) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + monkeypatch.delenv("DEEPSEEK_ANTHROPIC_API_BASE", raising=False) + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "deepseek-counted", + "litellm_params": {"model": "deepseek/deepseek-v4-pro", "api_key": "sk-deepseek-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post(_DEEPSEEK_COUNT_ROUTE).mock(return_value=httpx.Response(200, json={"input_tokens": 91})) + result: Final = await count_request_tokens( + request=TokenCountRequest(model="deepseek-counted", messages=[{"role": "user", "content": "Say hi."}]), + call_endpoint=True, + responses_request={ + "model": "deepseek-counted", + "input": "Say hi.", + "tools": [_WEATHER_TOOL], + **reasoning, + }, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert (result.total_tokens, result.count_source) == (91, "provider") + assert {field: sent[field] for field in ("thinking", "output_config") if field in sent} == expected_options + assert all("type" not in tool for tool in sent["tools"])