From 33b3aee5407e0a214afe91766d1c1fdd4dd888df Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 22 Apr 2026 14:21:06 +0800 Subject: [PATCH] up --- .../component/as_token_counter/estimate_token_counter.py | 8 ++++++-- {tests => test}/demo_memory_search.py | 0 {tests => test}/light/test_compactor.py | 0 {tests => test}/light/test_context_check.py | 0 {tests => test}/light/test_format_msgs_to_str.py | 0 {tests => test}/light/test_reme_light.py | 0 {tests => test}/light/test_reme_light_log.txt | 0 {tests => test}/light/test_summarizer.py | 0 {tests => test}/light/test_tool_result_compactor.py | 0 {tests => test}/light/test_tools.py | 0 {tests => test}/light/test_truncate_text_output.py | 0 {tests => test}/light/test_utils.py | 0 {tests => test}/mcp_servers_demo.json | 0 {tests => test}/test_agentscope_converter.py | 0 {tests => test}/test_base_context.py | 0 {tests => test}/test_base_file_watcher.py | 0 {tests => test}/test_cache_handler.py | 0 {tests => test}/test_cache_memory_usage.py | 0 {tests => test}/test_chunking_utils.py | 0 {tests => test}/test_embedding.py | 0 {tests => test}/test_embedding_cache.py | 0 {tests => test}/test_embedding_sync.py | 0 {tests => test}/test_execute_utils.py | 0 {tests => test}/test_file_store.py | 0 {tests => test}/test_fs_tool.py | 0 {tests => test}/test_horse.py | 0 {tests => test}/test_keyword_search_performance.py | 0 {tests => test}/test_llm.py | 0 {tests => test}/test_llm_sync.py | 0 {tests => test}/test_logo.py | 0 {tests => test}/test_mcp_client.py | 0 {tests => test}/test_mcp_server.py | 0 {tests => test}/test_memory_vector_conversion.py | 0 {tests => test}/test_message.py | 0 {tests => test}/test_reme.py | 0 {tests => test}/test_reme_memory_error_handling.py | 0 {tests => test}/test_timer.py | 0 {tests => test}/test_token_counter.py | 0 {tests => test}/test_tool.py | 0 {tests => test}/test_tool_call.py | 0 {tests => test}/test_ts_merge.py | 0 {tests => test}/test_vector_store.py | 0 {tests => test}/vector/test_reme_vector.py | 0 43 files changed, 6 insertions(+), 2 deletions(-) rename {tests => test}/demo_memory_search.py (100%) rename {tests => test}/light/test_compactor.py (100%) rename {tests => test}/light/test_context_check.py (100%) rename {tests => test}/light/test_format_msgs_to_str.py (100%) rename {tests => test}/light/test_reme_light.py (100%) rename {tests => test}/light/test_reme_light_log.txt (100%) rename {tests => test}/light/test_summarizer.py (100%) rename {tests => test}/light/test_tool_result_compactor.py (100%) rename {tests => test}/light/test_tools.py (100%) rename {tests => test}/light/test_truncate_text_output.py (100%) rename {tests => test}/light/test_utils.py (100%) rename {tests => test}/mcp_servers_demo.json (100%) rename {tests => test}/test_agentscope_converter.py (100%) rename {tests => test}/test_base_context.py (100%) rename {tests => test}/test_base_file_watcher.py (100%) rename {tests => test}/test_cache_handler.py (100%) rename {tests => test}/test_cache_memory_usage.py (100%) rename {tests => test}/test_chunking_utils.py (100%) rename {tests => test}/test_embedding.py (100%) rename {tests => test}/test_embedding_cache.py (100%) rename {tests => test}/test_embedding_sync.py (100%) rename {tests => test}/test_execute_utils.py (100%) rename {tests => test}/test_file_store.py (100%) rename {tests => test}/test_fs_tool.py (100%) rename {tests => test}/test_horse.py (100%) rename {tests => test}/test_keyword_search_performance.py (100%) rename {tests => test}/test_llm.py (100%) rename {tests => test}/test_llm_sync.py (100%) rename {tests => test}/test_logo.py (100%) rename {tests => test}/test_mcp_client.py (100%) rename {tests => test}/test_mcp_server.py (100%) rename {tests => test}/test_memory_vector_conversion.py (100%) rename {tests => test}/test_message.py (100%) rename {tests => test}/test_reme.py (100%) rename {tests => test}/test_reme_memory_error_handling.py (100%) rename {tests => test}/test_timer.py (100%) rename {tests => test}/test_token_counter.py (100%) rename {tests => test}/test_tool.py (100%) rename {tests => test}/test_tool_call.py (100%) rename {tests => test}/test_ts_merge.py (100%) rename {tests => test}/test_vector_store.py (100%) rename {tests => test}/vector/test_reme_vector.py (100%) diff --git a/reme2/component/as_token_counter/estimate_token_counter.py b/reme2/component/as_token_counter/estimate_token_counter.py index c36da7aa..b3ed881b 100644 --- a/reme2/component/as_token_counter/estimate_token_counter.py +++ b/reme2/component/as_token_counter/estimate_token_counter.py @@ -11,17 +11,21 @@ class EstimatedTokenCounter(TokenCounterBase): model's tokenizer directly. """ - def __init__(self, estimate_divisor: float = 4): + def __init__( + self, estimate_divisor: float = 4, encoding: str = "utf-8" + ): """Initialize the estimated token counter. Args: estimate_divisor: The divisor for character-to-token estimation. Default 4 assumes roughly 4 characters per token. Use 2-3 for Chinese/Japanese text, 4-5 for English. + encoding: The character encoding to use for byte length calculation. """ if estimate_divisor <= 0: raise ValueError("estimate_divisor cannot be zero") self.estimate_divisor: float = estimate_divisor + self.encoding: str = encoding async def count(self, text: str, **kwargs) -> int: """Count tokens in the given messages. @@ -35,4 +39,4 @@ class EstimatedTokenCounter(TokenCounterBase): """ if not text: return 0 - return int(len(text.encode("utf-8")) / self.estimate_divisor + 0.5) + return int(len(text.encode(self.encoding)) / self.estimate_divisor + 0.5) diff --git a/tests/demo_memory_search.py b/test/demo_memory_search.py similarity index 100% rename from tests/demo_memory_search.py rename to test/demo_memory_search.py diff --git a/tests/light/test_compactor.py b/test/light/test_compactor.py similarity index 100% rename from tests/light/test_compactor.py rename to test/light/test_compactor.py diff --git a/tests/light/test_context_check.py b/test/light/test_context_check.py similarity index 100% rename from tests/light/test_context_check.py rename to test/light/test_context_check.py diff --git a/tests/light/test_format_msgs_to_str.py b/test/light/test_format_msgs_to_str.py similarity index 100% rename from tests/light/test_format_msgs_to_str.py rename to test/light/test_format_msgs_to_str.py diff --git a/tests/light/test_reme_light.py b/test/light/test_reme_light.py similarity index 100% rename from tests/light/test_reme_light.py rename to test/light/test_reme_light.py diff --git a/tests/light/test_reme_light_log.txt b/test/light/test_reme_light_log.txt similarity index 100% rename from tests/light/test_reme_light_log.txt rename to test/light/test_reme_light_log.txt diff --git a/tests/light/test_summarizer.py b/test/light/test_summarizer.py similarity index 100% rename from tests/light/test_summarizer.py rename to test/light/test_summarizer.py diff --git a/tests/light/test_tool_result_compactor.py b/test/light/test_tool_result_compactor.py similarity index 100% rename from tests/light/test_tool_result_compactor.py rename to test/light/test_tool_result_compactor.py diff --git a/tests/light/test_tools.py b/test/light/test_tools.py similarity index 100% rename from tests/light/test_tools.py rename to test/light/test_tools.py diff --git a/tests/light/test_truncate_text_output.py b/test/light/test_truncate_text_output.py similarity index 100% rename from tests/light/test_truncate_text_output.py rename to test/light/test_truncate_text_output.py diff --git a/tests/light/test_utils.py b/test/light/test_utils.py similarity index 100% rename from tests/light/test_utils.py rename to test/light/test_utils.py diff --git a/tests/mcp_servers_demo.json b/test/mcp_servers_demo.json similarity index 100% rename from tests/mcp_servers_demo.json rename to test/mcp_servers_demo.json diff --git a/tests/test_agentscope_converter.py b/test/test_agentscope_converter.py similarity index 100% rename from tests/test_agentscope_converter.py rename to test/test_agentscope_converter.py diff --git a/tests/test_base_context.py b/test/test_base_context.py similarity index 100% rename from tests/test_base_context.py rename to test/test_base_context.py diff --git a/tests/test_base_file_watcher.py b/test/test_base_file_watcher.py similarity index 100% rename from tests/test_base_file_watcher.py rename to test/test_base_file_watcher.py diff --git a/tests/test_cache_handler.py b/test/test_cache_handler.py similarity index 100% rename from tests/test_cache_handler.py rename to test/test_cache_handler.py diff --git a/tests/test_cache_memory_usage.py b/test/test_cache_memory_usage.py similarity index 100% rename from tests/test_cache_memory_usage.py rename to test/test_cache_memory_usage.py diff --git a/tests/test_chunking_utils.py b/test/test_chunking_utils.py similarity index 100% rename from tests/test_chunking_utils.py rename to test/test_chunking_utils.py diff --git a/tests/test_embedding.py b/test/test_embedding.py similarity index 100% rename from tests/test_embedding.py rename to test/test_embedding.py diff --git a/tests/test_embedding_cache.py b/test/test_embedding_cache.py similarity index 100% rename from tests/test_embedding_cache.py rename to test/test_embedding_cache.py diff --git a/tests/test_embedding_sync.py b/test/test_embedding_sync.py similarity index 100% rename from tests/test_embedding_sync.py rename to test/test_embedding_sync.py diff --git a/tests/test_execute_utils.py b/test/test_execute_utils.py similarity index 100% rename from tests/test_execute_utils.py rename to test/test_execute_utils.py diff --git a/tests/test_file_store.py b/test/test_file_store.py similarity index 100% rename from tests/test_file_store.py rename to test/test_file_store.py diff --git a/tests/test_fs_tool.py b/test/test_fs_tool.py similarity index 100% rename from tests/test_fs_tool.py rename to test/test_fs_tool.py diff --git a/tests/test_horse.py b/test/test_horse.py similarity index 100% rename from tests/test_horse.py rename to test/test_horse.py diff --git a/tests/test_keyword_search_performance.py b/test/test_keyword_search_performance.py similarity index 100% rename from tests/test_keyword_search_performance.py rename to test/test_keyword_search_performance.py diff --git a/tests/test_llm.py b/test/test_llm.py similarity index 100% rename from tests/test_llm.py rename to test/test_llm.py diff --git a/tests/test_llm_sync.py b/test/test_llm_sync.py similarity index 100% rename from tests/test_llm_sync.py rename to test/test_llm_sync.py diff --git a/tests/test_logo.py b/test/test_logo.py similarity index 100% rename from tests/test_logo.py rename to test/test_logo.py diff --git a/tests/test_mcp_client.py b/test/test_mcp_client.py similarity index 100% rename from tests/test_mcp_client.py rename to test/test_mcp_client.py diff --git a/tests/test_mcp_server.py b/test/test_mcp_server.py similarity index 100% rename from tests/test_mcp_server.py rename to test/test_mcp_server.py diff --git a/tests/test_memory_vector_conversion.py b/test/test_memory_vector_conversion.py similarity index 100% rename from tests/test_memory_vector_conversion.py rename to test/test_memory_vector_conversion.py diff --git a/tests/test_message.py b/test/test_message.py similarity index 100% rename from tests/test_message.py rename to test/test_message.py diff --git a/tests/test_reme.py b/test/test_reme.py similarity index 100% rename from tests/test_reme.py rename to test/test_reme.py diff --git a/tests/test_reme_memory_error_handling.py b/test/test_reme_memory_error_handling.py similarity index 100% rename from tests/test_reme_memory_error_handling.py rename to test/test_reme_memory_error_handling.py diff --git a/tests/test_timer.py b/test/test_timer.py similarity index 100% rename from tests/test_timer.py rename to test/test_timer.py diff --git a/tests/test_token_counter.py b/test/test_token_counter.py similarity index 100% rename from tests/test_token_counter.py rename to test/test_token_counter.py diff --git a/tests/test_tool.py b/test/test_tool.py similarity index 100% rename from tests/test_tool.py rename to test/test_tool.py diff --git a/tests/test_tool_call.py b/test/test_tool_call.py similarity index 100% rename from tests/test_tool_call.py rename to test/test_tool_call.py diff --git a/tests/test_ts_merge.py b/test/test_ts_merge.py similarity index 100% rename from tests/test_ts_merge.py rename to test/test_ts_merge.py diff --git a/tests/test_vector_store.py b/test/test_vector_store.py similarity index 100% rename from tests/test_vector_store.py rename to test/test_vector_store.py diff --git a/tests/vector/test_reme_vector.py b/test/vector/test_reme_vector.py similarity index 100% rename from tests/vector/test_reme_vector.py rename to test/vector/test_reme_vector.py