mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
b9da8d2a7f
- Add new ToolCallResult and ToolMemory schemas for tracking tool executions - Implement tool memory retrieval and summarization flows in default config - Register new parse_tool_call_result_op for processing tool call results - Extend memory conversion logic to support tool memory type - Add test cases for tool memory serialization and deserialization - Include token counting utilities for text processing tasks
13 lines
374 B
Python
13 lines
374 B
Python
import tiktoken
|
||
|
||
def count_tokens(text: str) -> int:
|
||
"""计算给定文本在指定模型下的 token 数量"""
|
||
encoding = tiktoken.get_encoding("o200k_base")
|
||
tokens = encoding.encode(text)
|
||
return len(tokens)
|
||
|
||
# 示例使用
|
||
text = "你好，世界！Hello, world!"
|
||
token_count = count_tokens(text)
|
||
print(f"Token 数量: {token_count}")
|
||
print(len(text) / 4)
|