mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
2d181e6633
Local token counts for DeepSeek V3 and V4 model names now use DeepSeek's own bundled tokenizer instead of cl100k_base, and tool definitions on V4 names count with V4's chat-template tool framing
35 lines
1.3 KiB
Python
35 lines
1.3 KiB
Python
from collections.abc import Callable
|
|
from functools import cache
|
|
from importlib import resources
|
|
from typing import Final
|
|
|
|
from litellm.litellm_core_utils.default_encoding import encoding as cl100k_base
|
|
from tokenizers import Tokenizer
|
|
|
|
CHINESE_TEXT: Final = "深度求索的分词器会把这段中文切成更少的词元。" * 20
|
|
SHORT_TEXT: Final = "Find record 14."
|
|
|
|
|
|
@cache
|
|
def _deepseek_tokenizer() -> Tokenizer:
|
|
bundled: Final = resources.files("litellm.litellm_core_utils.tokenizers").joinpath("deepseek_tokenizer.json")
|
|
return Tokenizer.from_str(bundled.read_text(encoding="utf-8"))
|
|
|
|
|
|
def deepseek_tokens(text: str) -> int:
|
|
return len(_deepseek_tokenizer().encode(text).ids)
|
|
|
|
|
|
def cl100k_tokens(text: str) -> int:
|
|
return len(cl100k_base.encode(text))
|
|
|
|
|
|
def chinese_minus_short(tokens: Callable[[str], int]) -> int:
|
|
return tokens(CHINESE_TEXT) - tokens(SHORT_TEXT)
|
|
|
|
|
|
def assert_counted_with_deepseek(counted: int, expected: Callable[[Callable[[str], int]], int]) -> None:
|
|
cl100k_count: Final = expected(cl100k_tokens)
|
|
assert counted != cl100k_count, f"counted {counted}, the cl100k_base count, not DeepSeek's tokenizer"
|
|
deepseek_count: Final = expected(deepseek_tokens)
|
|
assert counted == deepseek_count, f"counted {counted}, DeepSeek's tokenizer counts {deepseek_count}"
|