mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
10 lines
279 B
Python
10 lines
279 B
Python
"""Token count estimation."""
|
|
|
|
|
|
def estimate_token_count(
|
|
text: str,
|
|
estimate_divisor: float = 4,
|
|
encoding: str = "utf-8",
|
|
) -> int:
|
|
"""Estimate the number of tokens in *text* by byte length."""
|
|
return int(len(text.encode(encoding)) / estimate_divisor + 0.5)
|