diff --git a/reme/agent/chat/fs_cli.py b/reme/agent/chat/fs_cli.py index e937b557..b1f61e68 100644 --- a/reme/agent/chat/fs_cli.py +++ b/reme/agent/chat/fs_cli.py @@ -10,20 +10,102 @@ from ...core.schema import Message, StreamChunk class FsCli(BaseReactStream): """FsCli agent with system prompt.""" - def __init__(self, working_dir: str, **kwargs): + def __init__( + self, + working_dir: str, + summary_params: dict | None = None, + compact_params: dict | None = None, + **kwargs, + ): super().__init__(**kwargs) self.working_dir: str = working_dir - self.messages: list[Message] = [] + self.summary_params: dict = summary_params or {} + self.compact_params: dict = compact_params or {} - def reset_history(self): - """Reset conversation history.""" + self.messages: list[Message] = [] + self.previous_summary: str = "" + + async def reset_history(self) -> str: + """Reset conversation history using summary. + + Summarizes current messages to memory files and clears history. + """ + if not self.messages: + self.messages.clear() + self.previous_summary = "" + return "No history to reset." + + # Import required modules + from ..fs import FsSummarizer + + # Summarize current conversation and save to memory files + current_date = datetime.now().strftime("%Y-%m-%d") + summarizer = FsSummarizer(tools=self.tools, **(self.summary_params or {})) + + result = await summarizer.call( + messages=self.messages, + date=current_date, + service_context=self.service_context, + ) + + # Clear messages (no previous_summary update, as summarizer saves to files) self.messages.clear() - return self + self.previous_summary = "" + + return f"History saved to memory files and reset. Result: {result.get('answer', 'Done')}" + + async def compact_history(self) -> str: + """Compact history then reset. + + First compacts messages if they exceed token limits (generating a summary), + then calls reset_history to save to files and clear. + """ + if not self.messages: + return "No history to compact." + + # Import required modules + from ..fs import FsCompactor + + # Step 1: Compact messages + compactor = FsCompactor(**(self.compact_params or {})) + compact_result = await compactor.call( + messages=self.messages, + previous_summary=self.previous_summary, + service_context=self.service_context, + ) + + compacted_messages = compact_result.get("messages", self.messages) + is_compacted = compact_result.get("compacted", False) + + if not is_compacted: + return "History is within token limits, no compaction needed." + + # Step 2: Extract summary from compacted messages + # The first message contains the summary wrapped in compaction_summary_format + tokens_before = compact_result.get("tokens_before", 0) + + if compacted_messages and compacted_messages[0].role == Role.USER: + # Extract summary content from the first message + summary_content = compacted_messages[0].content + self.previous_summary = summary_content + + # Step 3: Update messages and call reset_history to save and clear + self.messages = compacted_messages + reset_result = await self.reset_history() + + return f"History compacted from {tokens_before} tokens. {reset_result}" async def build_messages(self) -> list[Message]: """Build system prompt message.""" current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S %A") - system_prompt = self.prompt_format("system_prompt", workspace_dir=self.working_dir, current_time=current_time) + + system_prompt = self.prompt_format( + "system_prompt", + workspace_dir=self.working_dir, + current_time=current_time, + has_previous_summary=bool(self.previous_summary), + previous_summary=self.previous_summary or "", + ) return [ Message(role=Role.SYSTEM, content=system_prompt), diff --git a/reme/agent/chat/fs_cli.yaml b/reme/agent/chat/fs_cli.yaml index c39167ec..9ec8a90a 100644 --- a/reme/agent/chat/fs_cli.yaml +++ b/reme/agent/chat/fs_cli.yaml @@ -8,6 +8,13 @@ system_prompt: | Your working directory is: {workspace_dir} Treat this directory as the single global workspace for file operations unless explicitly instructed otherwise. + [has_previous_summary]## Previous Conversation Summary + [has_previous_summary] + [has_previous_summary]{previous_summary} + [has_previous_summary] + [has_previous_summary] + [has_previous_summary]The above is a summary of our previous conversation. Use it as context to maintain continuity. + ## Session Initialization Before doing anything else, read these files to orient yourself (don't ask permission): diff --git a/reme/agent/fs/fs_compactor.py b/reme/agent/fs/fs_compactor.py index 07c2dd18..2d379a60 100644 --- a/reme/agent/fs/fs_compactor.py +++ b/reme/agent/fs/fs_compactor.py @@ -17,12 +17,14 @@ class FsCompactor(BaseReact): context_window_tokens: int = 128000, reserve_tokens: int = 36000, keep_recent_tokens: int = 20000, + force_compact: bool = False, **kwargs, ): super().__init__(tools=[], **kwargs) self.context_window_tokens: int = context_window_tokens self.reserve_tokens: int = reserve_tokens self.keep_recent_tokens: int = keep_recent_tokens + self.force_compact: bool = force_compact @staticmethod def _normalize_messages(messages: list[Message | dict]) -> list[Message]: @@ -178,16 +180,20 @@ class FsCompactor(BaseReact): token_count: int = self.token_counter.count_token(original_messages) threshold = self.context_window_tokens - self.reserve_tokens - if token_count < threshold: + if not self.force_compact and token_count < threshold: logger.info(f"Token count {token_count} below threshold ({threshold}), skipping compaction") return { "compacted": False, "tokens_before": token_count, "is_split_turn": False, "messages": original_messages, + "summary_content": "", } - logger.info(f"Starting compaction, token count: {token_count}, threshold: {threshold}") + if self.force_compact: + logger.info(f"Force compaction enabled, token count: {token_count}, threshold: {threshold}") + else: + logger.info(f"Starting compaction, token count: {token_count}, threshold: {threshold}") history_prompt_messages = self.build_messages_s1() @@ -198,6 +204,7 @@ class FsCompactor(BaseReact): "tokens_before": token_count, "is_split_turn": False, "messages": original_messages, + "summary_content": "", } history_summary = await self._generate_summary(history_prompt_messages) if history_prompt_messages else "" @@ -222,4 +229,5 @@ class FsCompactor(BaseReact): "tokens_before": token_count, "is_split_turn": self.context.is_split_turn, "messages": final_messages, + "summary_content": summary_content, } diff --git a/reme/agent/fs/fs_summarizer.yaml b/reme/agent/fs/fs_summarizer.yaml index ef3de8ae..2e7b4d77 100644 --- a/reme/agent/fs/fs_summarizer.yaml +++ b/reme/agent/fs/fs_summarizer.yaml @@ -17,7 +17,23 @@ user_message_v2: | 1. Check if {memory_dir}/ exists; if not, create it via bash 2. Check if {memory_dir}/YYYY-MM-DD.md exists (use actual date) 3. If file is NEW: Write memories directly (be concise) - 4. If file EXISTS: Read it first, then UPDATE with new memories (keep concise, merge/deduplicate) + 4. If file EXISTS: + a) Read the existing file content + b) Compare conversation history with existing content + c) Identify NEW/UPDATED information not yet captured + d) Use edit_tool to add/update only the new information (preserve existing content) + e) If conversation contains NO new information, skip writing 5. If NO valuable information to store: Reply with reason and [SILENT] + IMPORTANT for updates: + - Only add information that is NOT already in the file + - Preserve all existing entries + - Merge duplicate information intelligently + - Use edit_tool for surgical updates, not write_tool (which overwrites) + + Example of what counts as NEW information: + - Existing: "Alice: Software engineer" + - Conversation: "Alice loves Python and AI projects" + - Action: ADD "Enjoys Python programming and AI project work" to Alice's entry + Store durable memories. Keep entries concise and well-organized. \ No newline at end of file diff --git a/reme/config/default.yaml b/reme/config/default.yaml index b7e913f3..3a7e7073 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -20,8 +20,8 @@ flows: llms: default: backend: openai -# model_name: qwen3-30b-a3b-instruct-2507 - model_name: qwen3-30b-a3b-thinking-2507 + model_name: qwen3-30b-a3b-instruct-2507 +# model_name: qwen3-30b-a3b-thinking-2507 request_interval: 1 # temperature: 0.0001 diff --git a/reme/config/fs.yaml b/reme/config/fs.yaml new file mode 100644 index 00000000..d32d6762 --- /dev/null +++ b/reme/config/fs.yaml @@ -0,0 +1,33 @@ +backend: cmd + +llms: + default: + backend: openai + model_name: qwen3-30b-a3b-instruct-2507 +# model_name: qwen3-30b-a3b-thinking-2507 + request_interval: 1 +# temperature: 0.0001 + +embedding_models: + default: + backend: openai + model_name: text-embedding-v4 + dimensions: 1024 + +memory_stores: + default: + backend: sqlite + store_name: test_hybrid + embedding_model: default + fts_enabled: true + snippet_max_chars: 700 + +token_counters: + default: + backend: base + + hf: + backend: hf + model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct + use_mirror: true + diff --git a/reme/core/application.py b/reme/core/application.py index d5ba08c2..9d8c051a 100644 --- a/reme/core/application.py +++ b/reme/core/application.py @@ -24,6 +24,7 @@ class Application: llm_api_base: str | None = None, embedding_api_key: str | None = None, embedding_api_base: str | None = None, + config_path: str | None = None, enable_logo: bool = True, log_to_console: bool = True, parser: type[PydanticConfigParser] | None = None, @@ -43,7 +44,7 @@ class Application: embedding_api_base=embedding_api_base, service_config=None, parser=parser, - config_path=None, + config_path=config_path, enable_logo=enable_logo, log_to_console=log_to_console, default_llm_config=default_llm_config, diff --git a/reme/reme.py b/reme/reme.py index 7094bbb0..abbddd43 100644 --- a/reme/reme.py +++ b/reme/reme.py @@ -51,6 +51,7 @@ class ReMe(Application): llm_api_base: str | None = None, embedding_api_key: str | None = None, embedding_api_base: str | None = None, + config_path: str = "default", enable_logo: bool = True, log_to_console: bool = True, default_llm_config: dict | None = None, @@ -71,6 +72,7 @@ class ReMe(Application): llm_api_base: API base for LLM provider embedding_api_key: API key for embedding provider embedding_api_base: API base for embedding provider + config_path: Path to config file enable_logo: Enable logo log_to_console: Log to console default_llm_config: LLM configuration @@ -102,6 +104,7 @@ class ReMe(Application): llm_api_base=llm_api_base, embedding_api_key=embedding_api_key, embedding_api_base=embedding_api_base, + config_path=config_path, enable_logo=enable_logo, log_to_console=log_to_console, parser=ReMeConfigParser, diff --git a/reme/reme_fs.py b/reme/reme_fs.py index 1d465a4d..2b61960b 100644 --- a/reme/reme_fs.py +++ b/reme/reme_fs.py @@ -12,7 +12,7 @@ from .agent.chat import FsCli from .agent.fs import FsCompactor, FsSummarizer from .config import ReMeConfigParser from .core import Application -from .core.enumeration import MemorySource, ChunkEnum +from .core.enumeration import ChunkEnum from .core.op import BaseTool from .core.schema import Message, StreamChunk from .tool.fs import ( @@ -38,6 +38,7 @@ class ReMeFs(Application): llm_api_base: str | None = None, embedding_api_key: str | None = None, embedding_api_base: str | None = None, + config_path: str = "fs", enable_logo: bool = True, log_to_console: bool = True, default_llm_config: dict | None = None, @@ -46,6 +47,9 @@ class ReMeFs(Application): default_token_counter_config: dict | None = None, default_file_watcher_config: dict | None = None, working_dir: str = ".reme", + compact_params: dict | None = None, + summary_params: dict | None = None, + search_params: dict | None = None, **kwargs, ): """Initialize ReMe with config.""" @@ -55,6 +59,7 @@ class ReMeFs(Application): llm_api_base=llm_api_base, embedding_api_key=embedding_api_key, embedding_api_base=embedding_api_base, + config_path=config_path, enable_logo=enable_logo, log_to_console=log_to_console, parser=ReMeConfigParser, @@ -67,6 +72,12 @@ class ReMeFs(Application): ) self.working_dir: str = working_dir + Path(self.working_dir).mkdir(parents=True, exist_ok=True) + self.compact_params: dict = compact_params or {} + self.summary_params: dict = summary_params or {} + self.search_params: dict = search_params or {} + + # Setup file system tools self.fs_tools: list[BaseTool] = [ BashTool(cwd=self.working_dir), EditTool(cwd=self.working_dir), @@ -76,72 +87,34 @@ class ReMeFs(Application): ReadTool(cwd=self.working_dir), WriteTool(cwd=self.working_dir), ] - self.working_path: Path = Path(self.working_dir) - self.working_path.mkdir(parents=True, exist_ok=True) + # Commands self.commands = [ "/new", + "/compact", "/exit", + "/help", ] - async def compact( - self, - messages: list[Message | dict], - context_window_tokens: int = 128000, - reserve_tokens: int = 36000, - keep_recent_tokens: int = 20000, - ): + async def compact(self, messages: list[Message | dict], previous_summary: str = ""): """Compact messages.""" messages = [Message(**message) if isinstance(message, dict) else message for message in messages] - compactor = FsCompactor( - context_window_tokens=context_window_tokens, - reserve_tokens=reserve_tokens, - keep_recent_tokens=keep_recent_tokens, + compactor = FsCompactor(**(self.compact_params or {})) + return await compactor.call( + messages=messages, + previous_summary=previous_summary, + service_context=self.service_context, ) - return await compactor.call(messages=messages, service_context=self.service_context) - - async def summary( - self, - messages: list[Message | dict], - date: str, - version: str = "default", - context_window_tokens: int = 128000, - reserve_tokens: int = 32000, - soft_threshold_tokens: int = 4000, - ): + async def summary(self, messages: list[Message | dict], date: str): """Summarize messages.""" messages = [Message(**message) if isinstance(message, dict) else message for message in messages] - summarizer = FsSummarizer( - tools=self.fs_tools, - version=version, - context_window_tokens=context_window_tokens, - reserve_tokens=reserve_tokens, - soft_threshold_tokens=soft_threshold_tokens, - ) - + summarizer = FsSummarizer(tools=self.fs_tools, **(self.summary_params or {})) return await summarizer.call(messages=messages, date=date, service_context=self.service_context) - async def memory_search( - self, - query: str, - max_results: int = 20, - min_score: float = 0.1, - sources: list[MemorySource] | None = None, - hybrid_enabled: bool = True, - hybrid_vector_weight: float = 0.7, - hybrid_text_weight: float = 0.3, - hybrid_candidate_multiplier: float = 3.0, - ) -> str: + async def memory_search(self, query: str, max_results: int = 10, min_score: float = 0.3) -> str: """Semantically search memory files.""" - search_tool = FsMemorySearch( - sources=sources, - hybrid_enabled=hybrid_enabled, - hybrid_vector_weight=hybrid_vector_weight, - hybrid_text_weight=hybrid_text_weight, - hybrid_candidate_multiplier=hybrid_candidate_multiplier, - ) - + search_tool = FsMemorySearch(**(self.search_params or {})) return await search_tool.call( query=query, max_results=max_results, @@ -156,7 +129,13 @@ class ReMeFs(Application): async def chat_with_remy(self, tool_result_max_size: int = 100): """Interactive CLI chat with Remy using simple streaming output.""" - fs_cli = FsCli(working_dir=self.working_dir, tools=self.fs_tools) + fs_cli = FsCli( + working_dir=self.working_dir, + tools=self.fs_tools, + summary_params=self.summary_params, + search_params=self.search_params, + compact_params=self.compact_params, + ) session = PromptSession() # Print welcome banner @@ -191,8 +170,19 @@ class ReMeFs(Application): break if user_input.strip() == "/new": - fs_cli.reset_history() - print("Conversation reset.\n") + result = await fs_cli.reset_history() + print(f"{result}\nConversation reset\n") + continue + + if user_input.strip() == "/compact": + result = await fs_cli.compact_history() + print(f"{result}\nHistory compacted.\n") + continue + + if user_input.strip() == "/help": + print("\nCommands:") + for command in self.commands: + print(f" {command}") continue # Stream processing state diff --git a/tests/test_fs_compact.py b/tests/test_fs_compact.py index 8824ea4c..af4b690f 100644 --- a/tests/test_fs_compact.py +++ b/tests/test_fs_compact.py @@ -118,7 +118,15 @@ async def test_compact_below_threshold(): print("TEST 1: Compact - Below Threshold (No Compaction)") print("=" * 80) - reme_fs = ReMeFs(enable_logo=False, vector_store=None) + reme_fs = ReMeFs( + enable_logo=False, + vector_store=None, + compact_params={ + "context_window_tokens": 5000, + "reserve_tokens": 2000, + "keep_recent_tokens": 1000, + }, + ) await reme_fs.start() messages = create_test_messages(num_messages=4) @@ -129,12 +137,7 @@ async def test_compact_below_threshold(): print(" reserve_tokens: 2000 (threshold = 3000)") print(" keep_recent_tokens: 1000") - result = await reme_fs.compact( - messages=messages, - context_window_tokens=5000, - reserve_tokens=2000, - keep_recent_tokens=1000, - ) + result = await reme_fs.compact(messages=messages) print(f"\n{'='*80}") print("RESULT:") @@ -161,7 +164,15 @@ async def test_compact_above_threshold(): print("TEST 2: Compact - Above Threshold (With Compaction & LLM Summary)") print("=" * 80) - reme_fs = ReMeFs(enable_logo=False, vector_store=None) + reme_fs = ReMeFs( + enable_logo=False, + vector_store=None, + compact_params={ + "context_window_tokens": 3000, + "reserve_tokens": 1500, + "keep_recent_tokens": 500, + }, + ) await reme_fs.start() messages = create_test_messages(num_messages=12) @@ -172,12 +183,7 @@ async def test_compact_above_threshold(): print(" reserve_tokens: 1500 (threshold = 1500)") print(" keep_recent_tokens: 500 (keep only recent messages)") - result = await reme_fs.compact( - messages=messages, - context_window_tokens=3000, - reserve_tokens=1500, - keep_recent_tokens=500, - ) + result = await reme_fs.compact(messages=messages) print(f"\n{'='*80}") print("RESULT:") @@ -208,7 +214,15 @@ async def test_compact_split_turn_scenario(): print("TEST 3: Compact - Split Turn Scenario (Cut in Middle of Assistant Response)") print("=" * 80) - reme_fs = ReMeFs(enable_logo=False, vector_store=None) + reme_fs = ReMeFs( + enable_logo=False, + vector_store=None, + compact_params={ + "context_window_tokens": 2000, + "reserve_tokens": 300, + "keep_recent_tokens": 600, + }, + ) await reme_fs.start() messages = [] @@ -242,16 +256,11 @@ async def test_compact_split_turn_scenario(): print_messages(messages, "INPUT MESSAGES", max_content_len=80) print("\nParameters:") - print(" context_window_tokens: 3000") - print(" reserve_tokens: 1000 (threshold = 2000)") - print(" keep_recent_tokens: 800 (should cut in middle of assistant responses)") + print(" context_window_tokens: 2000") + print(" reserve_tokens: 300 (threshold = 1700)") + print(" keep_recent_tokens: 600 (should cut in middle of assistant responses)") - result = await reme_fs.compact( - messages=messages, - context_window_tokens=3000, - reserve_tokens=1000, - keep_recent_tokens=800, - ) + result = await reme_fs.compact(messages=messages) print(f"\n{'='*80}") print("RESULT:") diff --git a/tests/test_fs_memory_search.py b/tests/test_fs_memory_search.py index cb88fa76..cea4bf42 100644 --- a/tests/test_fs_memory_search.py +++ b/tests/test_fs_memory_search.py @@ -340,13 +340,6 @@ async def test_memory_search_with_source_filter(): reme_fs = ReMeFs( enable_logo=False, working_dir=TestConfig.WORKING_DIR, - default_memory_store_config={ - "backend": "sqlite", - "store_name": "test_source_filter", - "embedding_model": "default", - "fts_enabled": True, - "snippet_max_chars": 700, - }, ) await reme_fs.start() @@ -382,11 +375,19 @@ async def test_memory_search_with_source_filter(): # Search only MEMORY source print(f"\n--- Searching MEMORY source for: '{query}' ---") - result_json = await reme_fs.memory_search( + # Create a new instance with MEMORY source filter + reme_fs_memory = ReMeFs( + enable_logo=False, + working_dir=TestConfig.WORKING_DIR, + search_params={"sources": [MemorySource.MEMORY]}, + ) + await reme_fs_memory.start() + result_json = await reme_fs_memory.memory_search( query=query, max_results=5, - sources=[MemorySource.MEMORY], ) + await reme_fs_memory.close() + import json memory_results = json.loads(result_json) @@ -396,11 +397,18 @@ async def test_memory_search_with_source_filter(): # Search only SESSIONS source print(f"\n--- Searching SESSIONS source for: '{query}' ---") - result_json = await reme_fs.memory_search( + # Create a new instance with SESSIONS source filter + reme_fs_sessions = ReMeFs( + enable_logo=False, + working_dir=TestConfig.WORKING_DIR, + search_params={"sources": [MemorySource.SESSIONS]}, + ) + await reme_fs_sessions.start() + result_json = await reme_fs_sessions.memory_search( query=query, max_results=5, - sources=[MemorySource.SESSIONS], ) + await reme_fs_sessions.close() session_results = json.loads(result_json) print(f"Found {len(session_results)} results in SESSIONS source") for result in session_results: @@ -597,13 +605,29 @@ async def test_memory_search_hybrid_mode(): # Test with hybrid enabled print(f"\n--- Hybrid search (enabled) for: '{query}' ---") - result_json_hybrid = await reme_fs.memory_search( + # Create instance with hybrid enabled + reme_fs_hybrid = ReMeFs( + enable_logo=False, + working_dir=TestConfig.WORKING_DIR, + default_memory_store_config={ + "backend": "sqlite", + "store_name": "test_hybrid", + "embedding_model": "default", + "fts_enabled": True, + "snippet_max_chars": 700, + }, + search_params={ + "hybrid_enabled": True, + "hybrid_vector_weight": 0.7, + "hybrid_text_weight": 0.3, + }, + ) + await reme_fs_hybrid.start() + result_json_hybrid = await reme_fs_hybrid.memory_search( query=query, max_results=5, - hybrid_enabled=True, - hybrid_vector_weight=0.7, - hybrid_text_weight=0.3, ) + await reme_fs_hybrid.close() import json @@ -613,11 +637,25 @@ async def test_memory_search_hybrid_mode(): # Test with hybrid disabled (vector only) print(f"\n--- Vector-only search for: '{query}' ---") - result_json_vector = await reme_fs.memory_search( + # Create instance with hybrid disabled + reme_fs_vector = ReMeFs( + enable_logo=False, + working_dir=TestConfig.WORKING_DIR, + default_memory_store_config={ + "backend": "sqlite", + "store_name": "test_hybrid", + "embedding_model": "default", + "fts_enabled": True, + "snippet_max_chars": 700, + }, + search_params={"hybrid_enabled": False}, + ) + await reme_fs_vector.start() + result_json_vector = await reme_fs_vector.memory_search( query=query, max_results=5, - hybrid_enabled=False, ) + await reme_fs_vector.close() vector_results = json.loads(result_json_vector) print(f"Vector search found {len(vector_results)} results") @@ -632,13 +670,29 @@ async def test_memory_search_hybrid_mode(): ] for vec_weight, text_weight in weight_configs: - result_json = await reme_fs.memory_search( + # Create instance with specific weights + reme_fs_weights = ReMeFs( + enable_logo=False, + working_dir=TestConfig.WORKING_DIR, + default_memory_store_config={ + "backend": "sqlite", + "store_name": "test_hybrid", + "embedding_model": "default", + "fts_enabled": True, + "snippet_max_chars": 700, + }, + search_params={ + "hybrid_enabled": True, + "hybrid_vector_weight": vec_weight, + "hybrid_text_weight": text_weight, + }, + ) + await reme_fs_weights.start() + result_json = await reme_fs_weights.memory_search( query=query, max_results=5, - hybrid_enabled=True, - hybrid_vector_weight=vec_weight, - hybrid_text_weight=text_weight, ) + await reme_fs_weights.close() results = json.loads(result_json) print(f" Vector:{vec_weight}/Text:{text_weight} -> {len(results)} results") diff --git a/tests/test_fs_summary.py b/tests/test_fs_summary.py index ff9ec210..6cf7798c 100644 --- a/tests/test_fs_summary.py +++ b/tests/test_fs_summary.py @@ -155,7 +155,6 @@ async def test_summary_personal_info_storage(): result = await reme_fs.summary( messages=messages, - version="default", date="2023-09-01", ) @@ -191,7 +190,6 @@ async def test_summary_detailed_profile(): result = await reme_fs.summary( messages=messages, - version="default", date="2023-10-01", )