mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
7 lines
165 B
Python
7 lines
165 B
Python
import tiktoken
|
|
|
|
enc = tiktoken.get_encoding("o200k_base")
|
|
|
|
# r = enc.encode("我爱吃西瓜，你说啥")
|
|
r = enc.encode("hello world aaaaaaaaaaaa")
|
|
print(len(r))
|