diff --git a/reme_old_doc/__init__.py b/reme_old_doc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reme/extension/cli/__init__.py b/test/cli/__init__.py similarity index 100% rename from reme/extension/cli/__init__.py rename to test/cli/__init__.py diff --git a/reme/extension/cli/fb_cli.py b/test/cli/fb_cli.py similarity index 100% rename from reme/extension/cli/fb_cli.py rename to test/cli/fb_cli.py diff --git a/reme/extension/cli/fb_cli.yaml b/test/cli/fb_cli.yaml similarity index 100% rename from reme/extension/cli/fb_cli.yaml rename to test/cli/fb_cli.yaml diff --git a/reme/extension/cli/fb_compactor.py b/test/cli/fb_compactor.py similarity index 100% rename from reme/extension/cli/fb_compactor.py rename to test/cli/fb_compactor.py diff --git a/reme/extension/cli/fb_compactor.yaml b/test/cli/fb_compactor.yaml similarity index 100% rename from reme/extension/cli/fb_compactor.yaml rename to test/cli/fb_compactor.yaml diff --git a/reme/extension/cli/fb_context_checker.py b/test/cli/fb_context_checker.py similarity index 100% rename from reme/extension/cli/fb_context_checker.py rename to test/cli/fb_context_checker.py diff --git a/reme/extension/cli/fb_summarizer.py b/test/cli/fb_summarizer.py similarity index 100% rename from reme/extension/cli/fb_summarizer.py rename to test/cli/fb_summarizer.py diff --git a/reme/extension/cli/fb_summarizer.yaml b/test/cli/fb_summarizer.yaml similarity index 100% rename from reme/extension/cli/fb_summarizer.yaml rename to test/cli/fb_summarizer.yaml diff --git a/reme/extension/reme_cli.py b/test/reme_cli.py similarity index 100% rename from reme/extension/reme_cli.py rename to test/reme_cli.py diff --git a/tests/light/test_context_check.py b/tests/light/test_context_check.py index f65f961e..872d6e2d 100644 --- a/tests/light/test_context_check.py +++ b/tests/light/test_context_check.py @@ -220,7 +220,7 @@ def test_empty_messages(): handler = create_handler() messages = [] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -240,7 +240,7 @@ def test_below_threshold_returns_all(): create_user_msg("How are you?"), ] threshold, reserve = 10000, 5000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Very high threshold memory_compact_reserve=reserve, @@ -271,7 +271,7 @@ def test_above_threshold_triggers_compaction(): create_assistant_msg("Fourth message " * 100), ] threshold, reserve = 100, 200 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Low threshold to trigger compaction memory_compact_reserve=reserve, @@ -302,7 +302,7 @@ def test_message_order_preserved(): create_user_msg("Fifth " * 10), ] threshold, reserve = 100, 150 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Low threshold memory_compact_reserve=reserve, @@ -333,7 +333,7 @@ def test_single_message_below_threshold(): handler = create_handler() messages = [create_user_msg("Short message")] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -358,7 +358,7 @@ def test_single_message_above_threshold(): long_content = "Very long message " * 1000 messages = [create_user_msg(long_content)] threshold, reserve = 10, 5 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Very low threshold memory_compact_reserve=reserve, # Even lower reserve @@ -386,7 +386,7 @@ def test_reserve_zero(): create_assistant_msg("Hi there!"), ] threshold, reserve = 1, 0 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Trigger compaction memory_compact_reserve=reserve, # Zero reserve @@ -403,7 +403,7 @@ def test_threshold_zero(): handler = create_handler() messages = [create_user_msg("A")] # Minimal message threshold, reserve = 0, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Zero threshold - always triggers memory_compact_reserve=reserve, @@ -426,7 +426,7 @@ def test_exact_threshold_boundary(): threshold, reserve = exact_tokens, exact_tokens # Test at exact boundary - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Exactly at boundary memory_compact_reserve=reserve, @@ -454,7 +454,7 @@ def test_reserve_larger_than_threshold(): create_assistant_msg("Message two " * 20), ] threshold, reserve = 50, 10000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Low threshold memory_compact_reserve=reserve, # High reserve @@ -489,7 +489,7 @@ def test_tool_use_result_paired(): create_assistant_msg("The tool returned results"), ] threshold, reserve = 50, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Trigger compaction memory_compact_reserve=reserve, # Enough for tool pair @@ -522,7 +522,7 @@ def test_tool_use_without_result(): create_assistant_msg("Something happened"), ] threshold, reserve = 10, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -550,7 +550,7 @@ def test_tool_result_without_use(): create_assistant_msg("Got it"), ] threshold, reserve = 10, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -583,7 +583,7 @@ def test_multiple_tool_pairs(): create_assistant_msg("All done"), ] threshold, reserve = 50, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -630,7 +630,7 @@ def test_tool_dependency_causes_extra_inclusion(): create_assistant_msg("End"), # Small ] threshold, reserve = 100, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Trigger compaction memory_compact_reserve=reserve, # Medium reserve @@ -671,7 +671,7 @@ def test_tool_dependency_exceeds_reserve(): create_assistant_msg("Last message"), ] threshold, reserve = 10, 100 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Trigger compaction memory_compact_reserve=reserve, # Small reserve - can't fit the pair @@ -715,7 +715,7 @@ def test_interleaved_tool_pairs(): create_assistant_msg("Both done"), ] threshold, reserve = 50, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -756,7 +756,7 @@ def test_message_with_empty_content(): create_assistant_msg("Response"), ] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -782,7 +782,7 @@ def test_message_with_whitespace_only(): create_assistant_msg("Response"), ] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -806,7 +806,7 @@ def test_very_long_single_message(): huge_content = "x" * 100000 # Very long messages = [create_user_msg(huge_content)] threshold, reserve = 100, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -830,7 +830,7 @@ def test_many_small_messages(): handler = create_handler() messages = [create_user_msg(f"Msg {i}") for i in range(100)] threshold, reserve = 100, 200 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Low threshold memory_compact_reserve=reserve, @@ -859,7 +859,7 @@ def test_unicode_content(): create_user_msg("日本語テスト 🇯🇵"), ] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -877,7 +877,7 @@ def test_special_characters_content(): create_assistant_msg("More: \n\r\t\0 nulls and newlines"), ] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -912,7 +912,7 @@ def test_all_messages_fit_exactly_in_reserve(): total = sum(handler.stat_message(m).total_tokens for m in messages) threshold, reserve = total - 1, total - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Just below total to trigger memory_compact_reserve=reserve, # Exactly fits all @@ -945,7 +945,7 @@ def test_first_message_only_compacted(): tiny_msg_tokens = handler.stat_message(messages[2]).total_tokens threshold, reserve = 50, small_msg_tokens + tiny_msg_tokens + 10 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Low to trigger memory_compact_reserve=reserve, # Fits last 2 @@ -976,7 +976,7 @@ def test_last_message_only_kept(): tiny_tokens = handler.stat_message(messages[2]).total_tokens threshold, reserve = 10, tiny_tokens + 5 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, # Only fits last message @@ -1005,7 +1005,7 @@ def test_all_messages_compacted(): create_assistant_msg("Large message " * 100), ] threshold, reserve = 10, 1 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, # Trigger compaction memory_compact_reserve=reserve, # Too small for anything @@ -1039,7 +1039,7 @@ def test_system_message(): create_assistant_msg("Hi there!"), ] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -1061,7 +1061,7 @@ def test_mixed_roles(): Msg(name="helper", role="assistant", content="Another assistant message"), ] threshold, reserve = 1000, 500 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -1085,7 +1085,7 @@ def test_tool_use_with_empty_id(): create_assistant_msg("Done"), ] threshold, reserve = 10, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -1113,7 +1113,7 @@ def test_tool_result_with_empty_id(): create_assistant_msg("Noted"), ] threshold, reserve = 10, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -1142,7 +1142,7 @@ def test_duplicate_tool_ids(): create_tool_result_msg("call_dup", "tool_b", "Result B"), ] threshold, reserve = 10, 1000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, @@ -1181,7 +1181,7 @@ def test_message_with_multiple_tool_blocks(): create_tool_result_msg("call_3", "tool3", "Result 3"), ] threshold, reserve = 10, 2000 - to_compact, to_keep = handler.context_check( + to_compact, to_keep, _ = handler.context_check( messages=messages, memory_compact_threshold=threshold, memory_compact_reserve=reserve, diff --git a/tests/light/test_format_msgs_to_str.py b/tests/light/test_format_msgs_to_str.py index bd69751a..93dd7a2f 100644 --- a/tests/light/test_format_msgs_to_str.py +++ b/tests/light/test_format_msgs_to_str.py @@ -481,10 +481,11 @@ def test_format_msgs_to_str_threshold_zero(): def test_format_msgs_to_str_threshold_exact_fit(): """Test when messages exactly fit the threshold.""" handler = create_handler() - # Create a message and measure its tokens + # Create a message and measure its formatted string tokens msg = create_user_msg("Test") stat = handler.stat_message(msg) - exact_threshold = stat.total_tokens + formatted_content = stat.format(include_thinking=False) + exact_threshold = handler.count_str_token(formatted_content) msgs = [msg] result = handler.format_msgs_to_str(msgs, memory_compact_threshold=exact_threshold) @@ -499,7 +500,8 @@ def test_format_msgs_to_str_threshold_one_less(): handler = create_handler() msg = create_user_msg("Test message") stat = handler.stat_message(msg) - threshold_minus_one = stat.total_tokens - 1 + formatted_content = stat.format(include_thinking=False) + threshold_minus_one = handler.count_str_token(formatted_content) - 1 msgs = [msg] result = handler.format_msgs_to_str(msgs, memory_compact_threshold=threshold_minus_one)