scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-08-28 05:25:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

feat(core): add config path parameter and enhance fs cli capabilities

Browse source
This commit is contained in:
jinli.yl 2026-02-08 04:56:10 +08:00
parent f0bc2da7b0
commit 1a35150648
12 changed files with 315 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
reme/agent/chat/fs_cli.py
View file

@ -10,20 +10,102 @@ from ...core.schema import Message, StreamChunk
class FsCli(BaseReactStream):
"""FsCli agent with system prompt."""
def __init__(self, working_dir: str, **kwargs):
def __init__(
self,
working_dir: str,
summary_params: dict | None = None,
compact_params: dict | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.working_dir: str = working_dir
self.messages: list[Message] = []
self.summary_params: dict = summary_params or {}
self.compact_params: dict = compact_params or {}
def reset_history(self):
"""Reset conversation history."""
self.messages: list[Message] = []
self.previous_summary: str = ""
async def reset_history(self) -> str:
"""Reset conversation history using summary.
Summarizes current messages to memory files and clears history.
"""
if not self.messages:
self.messages.clear()
self.previous_summary = ""
return "No history to reset."
# Import required modules
from ..fs import FsSummarizer
# Summarize current conversation and save to memory files
current_date = datetime.now().strftime("%Y-%m-%d")
summarizer = FsSummarizer(tools=self.tools, **(self.summary_params or {}))
result = await summarizer.call(
messages=self.messages,
date=current_date,
service_context=self.service_context,
)
# Clear messages (no previous_summary update, as summarizer saves to files)
self.messages.clear()
return self
self.previous_summary = ""
return f"History saved to memory files and reset. Result: {result.get('answer', 'Done')}"
async def compact_history(self) -> str:
"""Compact history then reset.
First compacts messages if they exceed token limits (generating a summary),
then calls reset_history to save to files and clear.
"""
if not self.messages:
return "No history to compact."
# Import required modules
from ..fs import FsCompactor
# Step 1: Compact messages
compactor = FsCompactor(**(self.compact_params or {}))
compact_result = await compactor.call(
messages=self.messages,
previous_summary=self.previous_summary,
service_context=self.service_context,
)
compacted_messages = compact_result.get("messages", self.messages)
is_compacted = compact_result.get("compacted", False)
if not is_compacted:
return "History is within token limits, no compaction needed."
# Step 2: Extract summary from compacted messages
# The first message contains the summary wrapped in compaction_summary_format
tokens_before = compact_result.get("tokens_before", 0)
if compacted_messages and compacted_messages[0].role == Role.USER:
# Extract summary content from the first message
summary_content = compacted_messages[0].content
self.previous_summary = summary_content
# Step 3: Update messages and call reset_history to save and clear
self.messages = compacted_messages
reset_result = await self.reset_history()
return f"History compacted from {tokens_before} tokens. {reset_result}"
async def build_messages(self) -> list[Message]:
"""Build system prompt message."""
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S %A")
system_prompt = self.prompt_format("system_prompt", workspace_dir=self.working_dir, current_time=current_time)
system_prompt = self.prompt_format(
"system_prompt",
workspace_dir=self.working_dir,
current_time=current_time,
has_previous_summary=bool(self.previous_summary),
previous_summary=self.previous_summary or "",
)
return [
Message(role=Role.SYSTEM, content=system_prompt),

7
reme/agent/chat/fs_cli.yaml
View file

@ -8,6 +8,13 @@ system_prompt: |
Your working directory is: {workspace_dir}
Treat this directory as the single global workspace for file operations unless explicitly instructed otherwise.
[has_previous_summary]## Previous Conversation Summary
[has_previous_summary]<previous-summary>
[has_previous_summary]{previous_summary}
[has_previous_summary]</previous-summary>
[has_previous_summary]
[has_previous_summary]The above is a summary of our previous conversation. Use it as context to maintain continuity.
## Session Initialization
Before doing anything else, read these files to orient yourself (don't ask permission):

12
reme/agent/fs/fs_compactor.py
View file

@ -17,12 +17,14 @@ class FsCompactor(BaseReact):
context_window_tokens: int = 128000,
reserve_tokens: int = 36000,
keep_recent_tokens: int = 20000,
force_compact: bool = False,
**kwargs,
):
super().__init__(tools=[], **kwargs)
self.context_window_tokens: int = context_window_tokens
self.reserve_tokens: int = reserve_tokens
self.keep_recent_tokens: int = keep_recent_tokens
self.force_compact: bool = force_compact
@staticmethod
def _normalize_messages(messages: list[Message | dict]) -> list[Message]:
@ -178,16 +180,20 @@ class FsCompactor(BaseReact):
token_count: int = self.token_counter.count_token(original_messages)
threshold = self.context_window_tokens - self.reserve_tokens
if token_count < threshold:
if not self.force_compact and token_count < threshold:
logger.info(f"Token count {token_count} below threshold ({threshold}), skipping compaction")
return {
"compacted": False,
"tokens_before": token_count,
"is_split_turn": False,
"messages": original_messages,
"summary_content": "",
}
logger.info(f"Starting compaction, token count: {token_count}, threshold: {threshold}")
if self.force_compact:
logger.info(f"Force compaction enabled, token count: {token_count}, threshold: {threshold}")
else:
logger.info(f"Starting compaction, token count: {token_count}, threshold: {threshold}")
history_prompt_messages = self.build_messages_s1()
@ -198,6 +204,7 @@ class FsCompactor(BaseReact):
"tokens_before": token_count,
"is_split_turn": False,
"messages": original_messages,
"summary_content": "",
}
history_summary = await self._generate_summary(history_prompt_messages) if history_prompt_messages else ""
@ -222,4 +229,5 @@ class FsCompactor(BaseReact):
"tokens_before": token_count,
"is_split_turn": self.context.is_split_turn,
"messages": final_messages,
"summary_content": summary_content,
}

18
reme/agent/fs/fs_summarizer.yaml
View file

@ -17,7 +17,23 @@ user_message_v2: |
1. Check if {memory_dir}/ exists; if not, create it via bash
2. Check if {memory_dir}/YYYY-MM-DD.md exists (use actual date)
3. If file is NEW: Write memories directly (be concise)
4. If file EXISTS: Read it first, then UPDATE with new memories (keep concise, merge/deduplicate)
4. If file EXISTS:
a) Read the existing file content
b) Compare conversation history with existing content
c) Identify NEW/UPDATED information not yet captured
d) Use edit_tool to add/update only the new information (preserve existing content)
e) If conversation contains NO new information, skip writing
5. If NO valuable information to store: Reply with reason and [SILENT]
IMPORTANT for updates:
- Only add information that is NOT already in the file
- Preserve all existing entries
- Merge duplicate information intelligently
- Use edit_tool for surgical updates, not write_tool (which overwrites)
Example of what counts as NEW information:
- Existing: "Alice: Software engineer"
- Conversation: "Alice loves Python and AI projects"
- Action: ADD "Enjoys Python programming and AI project work" to Alice's entry
Store durable memories. Keep entries concise and well-organized.

4
reme/config/default.yaml
View file

@ -20,8 +20,8 @@ flows:
llms:
default:
backend: openai
# model_name: qwen3-30b-a3b-instruct-2507
model_name: qwen3-30b-a3b-thinking-2507
model_name: qwen3-30b-a3b-instruct-2507
# model_name: qwen3-30b-a3b-thinking-2507
request_interval: 1
# temperature: 0.0001

33
reme/config/fs.yaml Normal file
View file

@ -0,0 +1,33 @@
backend: cmd
llms:
default:
backend: openai
model_name: qwen3-30b-a3b-instruct-2507
# model_name: qwen3-30b-a3b-thinking-2507
request_interval: 1
# temperature: 0.0001
embedding_models:
default:
backend: openai
model_name: text-embedding-v4
dimensions: 1024
memory_stores:
default:
backend: sqlite
store_name: test_hybrid
embedding_model: default
fts_enabled: true
snippet_max_chars: 700
token_counters:
default:
backend: base
hf:
backend: hf
model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct
use_mirror: true

3
reme/core/application.py
View file

@ -24,6 +24,7 @@ class Application:
llm_api_base: str | None = None,
embedding_api_key: str | None = None,
embedding_api_base: str | None = None,
config_path: str | None = None,
enable_logo: bool = True,
log_to_console: bool = True,
parser: type[PydanticConfigParser] | None = None,
@ -43,7 +44,7 @@ class Application:
embedding_api_base=embedding_api_base,
service_config=None,
parser=parser,
config_path=None,
config_path=config_path,
enable_logo=enable_logo,
log_to_console=log_to_console,
default_llm_config=default_llm_config,

3
reme/reme.py
View file

@ -51,6 +51,7 @@ class ReMe(Application):
llm_api_base: str | None = None,
embedding_api_key: str | None = None,
embedding_api_base: str | None = None,
config_path: str = "default",
enable_logo: bool = True,
log_to_console: bool = True,
default_llm_config: dict | None = None,
@ -71,6 +72,7 @@ class ReMe(Application):
llm_api_base: API base for LLM provider
embedding_api_key: API key for embedding provider
embedding_api_base: API base for embedding provider
config_path: Path to config file
enable_logo: Enable logo
log_to_console: Log to console
default_llm_config: LLM configuration
@ -102,6 +104,7 @@ class ReMe(Application):
llm_api_base=llm_api_base,
embedding_api_key=embedding_api_key,
embedding_api_base=embedding_api_base,
config_path=config_path,
enable_logo=enable_logo,
log_to_console=log_to_console,
parser=ReMeConfigParser,

100
reme/reme_fs.py
View file

@ -12,7 +12,7 @@ from .agent.chat import FsCli
from .agent.fs import FsCompactor, FsSummarizer
from .config import ReMeConfigParser
from .core import Application
from .core.enumeration import MemorySource, ChunkEnum
from .core.enumeration import ChunkEnum
from .core.op import BaseTool
from .core.schema import Message, StreamChunk
from .tool.fs import (
@ -38,6 +38,7 @@ class ReMeFs(Application):
llm_api_base: str | None = None,
embedding_api_key: str | None = None,
embedding_api_base: str | None = None,
config_path: str = "fs",
enable_logo: bool = True,
log_to_console: bool = True,
default_llm_config: dict | None = None,
@ -46,6 +47,9 @@ class ReMeFs(Application):
default_token_counter_config: dict | None = None,
default_file_watcher_config: dict | None = None,
working_dir: str = ".reme",
compact_params: dict | None = None,
summary_params: dict | None = None,
search_params: dict | None = None,
**kwargs,
):
"""Initialize ReMe with config."""
@ -55,6 +59,7 @@ class ReMeFs(Application):
llm_api_base=llm_api_base,
embedding_api_key=embedding_api_key,
embedding_api_base=embedding_api_base,
config_path=config_path,
enable_logo=enable_logo,
log_to_console=log_to_console,
parser=ReMeConfigParser,
@ -67,6 +72,12 @@ class ReMeFs(Application):
)
self.working_dir: str = working_dir
Path(self.working_dir).mkdir(parents=True, exist_ok=True)
self.compact_params: dict = compact_params or {}
self.summary_params: dict = summary_params or {}
self.search_params: dict = search_params or {}
# Setup file system tools
self.fs_tools: list[BaseTool] = [
BashTool(cwd=self.working_dir),
EditTool(cwd=self.working_dir),
@ -76,72 +87,34 @@ class ReMeFs(Application):
ReadTool(cwd=self.working_dir),
WriteTool(cwd=self.working_dir),
]
self.working_path: Path = Path(self.working_dir)
self.working_path.mkdir(parents=True, exist_ok=True)
# Commands
self.commands = [
"/new",
"/compact",
"/exit",
"/help",
]
async def compact(
self,
messages: list[Message | dict],
context_window_tokens: int = 128000,
reserve_tokens: int = 36000,
keep_recent_tokens: int = 20000,
):
async def compact(self, messages: list[Message | dict], previous_summary: str = ""):
"""Compact messages."""
messages = [Message(**message) if isinstance(message, dict) else message for message in messages]
compactor = FsCompactor(
context_window_tokens=context_window_tokens,
reserve_tokens=reserve_tokens,
keep_recent_tokens=keep_recent_tokens,
compactor = FsCompactor(**(self.compact_params or {}))
return await compactor.call(
messages=messages,
previous_summary=previous_summary,
service_context=self.service_context,
)
return await compactor.call(messages=messages, service_context=self.service_context)
async def summary(
self,
messages: list[Message | dict],
date: str,
version: str = "default",
context_window_tokens: int = 128000,
reserve_tokens: int = 32000,
soft_threshold_tokens: int = 4000,
):
async def summary(self, messages: list[Message | dict], date: str):
"""Summarize messages."""
messages = [Message(**message) if isinstance(message, dict) else message for message in messages]
summarizer = FsSummarizer(
tools=self.fs_tools,
version=version,
context_window_tokens=context_window_tokens,
reserve_tokens=reserve_tokens,
soft_threshold_tokens=soft_threshold_tokens,
)
summarizer = FsSummarizer(tools=self.fs_tools, **(self.summary_params or {}))
return await summarizer.call(messages=messages, date=date, service_context=self.service_context)
async def memory_search(
self,
query: str,
max_results: int = 20,
min_score: float = 0.1,
sources: list[MemorySource] | None = None,
hybrid_enabled: bool = True,
hybrid_vector_weight: float = 0.7,
hybrid_text_weight: float = 0.3,
hybrid_candidate_multiplier: float = 3.0,
) -> str:
async def memory_search(self, query: str, max_results: int = 10, min_score: float = 0.3) -> str:
"""Semantically search memory files."""
search_tool = FsMemorySearch(
sources=sources,
hybrid_enabled=hybrid_enabled,
hybrid_vector_weight=hybrid_vector_weight,
hybrid_text_weight=hybrid_text_weight,
hybrid_candidate_multiplier=hybrid_candidate_multiplier,
)
search_tool = FsMemorySearch(**(self.search_params or {}))
return await search_tool.call(
query=query,
max_results=max_results,
@ -156,7 +129,13 @@ class ReMeFs(Application):
async def chat_with_remy(self, tool_result_max_size: int = 100):
"""Interactive CLI chat with Remy using simple streaming output."""
fs_cli = FsCli(working_dir=self.working_dir, tools=self.fs_tools)
fs_cli = FsCli(
working_dir=self.working_dir,
tools=self.fs_tools,
summary_params=self.summary_params,
search_params=self.search_params,
compact_params=self.compact_params,
)
session = PromptSession()
# Print welcome banner
@ -191,8 +170,19 @@ class ReMeFs(Application):
break
if user_input.strip() == "/new":
fs_cli.reset_history()
print("Conversation reset.\n")
result = await fs_cli.reset_history()
print(f"{result}\nConversation reset\n")
continue
if user_input.strip() == "/compact":
result = await fs_cli.compact_history()
print(f"{result}\nHistory compacted.\n")
continue
if user_input.strip() == "/help":
print("\nCommands:")
for command in self.commands:
print(f" {command}")
continue
# Stream processing state

57
tests/test_fs_compact.py
View file

@ -118,7 +118,15 @@ async def test_compact_below_threshold():
print("TEST 1: Compact - Below Threshold (No Compaction)")
print("=" * 80)
reme_fs = ReMeFs(enable_logo=False, vector_store=None)
reme_fs = ReMeFs(
enable_logo=False,
vector_store=None,
compact_params={
"context_window_tokens": 5000,
"reserve_tokens": 2000,
"keep_recent_tokens": 1000,
},
)
await reme_fs.start()
messages = create_test_messages(num_messages=4)
@ -129,12 +137,7 @@ async def test_compact_below_threshold():
print(" reserve_tokens: 2000 (threshold = 3000)")
print(" keep_recent_tokens: 1000")
result = await reme_fs.compact(
messages=messages,
context_window_tokens=5000,
reserve_tokens=2000,
keep_recent_tokens=1000,
)
result = await reme_fs.compact(messages=messages)
print(f"\n{'='*80}")
print("RESULT:")
@ -161,7 +164,15 @@ async def test_compact_above_threshold():
print("TEST 2: Compact - Above Threshold (With Compaction & LLM Summary)")
print("=" * 80)
reme_fs = ReMeFs(enable_logo=False, vector_store=None)
reme_fs = ReMeFs(
enable_logo=False,
vector_store=None,
compact_params={
"context_window_tokens": 3000,
"reserve_tokens": 1500,
"keep_recent_tokens": 500,
},
)
await reme_fs.start()
messages = create_test_messages(num_messages=12)
@ -172,12 +183,7 @@ async def test_compact_above_threshold():
print(" reserve_tokens: 1500 (threshold = 1500)")
print(" keep_recent_tokens: 500 (keep only recent messages)")
result = await reme_fs.compact(
messages=messages,
context_window_tokens=3000,
reserve_tokens=1500,
keep_recent_tokens=500,
)
result = await reme_fs.compact(messages=messages)
print(f"\n{'='*80}")
print("RESULT:")
@ -208,7 +214,15 @@ async def test_compact_split_turn_scenario():
print("TEST 3: Compact - Split Turn Scenario (Cut in Middle of Assistant Response)")
print("=" * 80)
reme_fs = ReMeFs(enable_logo=False, vector_store=None)
reme_fs = ReMeFs(
enable_logo=False,
vector_store=None,
compact_params={
"context_window_tokens": 2000,
"reserve_tokens": 300,
"keep_recent_tokens": 600,
},
)
await reme_fs.start()
messages = []
@ -242,16 +256,11 @@ async def test_compact_split_turn_scenario():
print_messages(messages, "INPUT MESSAGES", max_content_len=80)
print("\nParameters:")
print(" context_window_tokens: 3000")
print(" reserve_tokens: 1000 (threshold = 2000)")
print(" keep_recent_tokens: 800 (should cut in middle of assistant responses)")
print(" context_window_tokens: 2000")
print(" reserve_tokens: 300 (threshold = 1700)")
print(" keep_recent_tokens: 600 (should cut in middle of assistant responses)")
result = await reme_fs.compact(
messages=messages,
context_window_tokens=3000,
reserve_tokens=1000,
keep_recent_tokens=800,
)
result = await reme_fs.compact(messages=messages)
print(f"\n{'='*80}")
print("RESULT:")

96
tests/test_fs_memory_search.py
View file

@ -340,13 +340,6 @@ async def test_memory_search_with_source_filter():
reme_fs = ReMeFs(
enable_logo=False,
working_dir=TestConfig.WORKING_DIR,
default_memory_store_config={
"backend": "sqlite",
"store_name": "test_source_filter",
"embedding_model": "default",
"fts_enabled": True,
"snippet_max_chars": 700,
},
)
await reme_fs.start()
@ -382,11 +375,19 @@ async def test_memory_search_with_source_filter():
# Search only MEMORY source
print(f"\n--- Searching MEMORY source for: '{query}' ---")
result_json = await reme_fs.memory_search(
# Create a new instance with MEMORY source filter
reme_fs_memory = ReMeFs(
enable_logo=False,
working_dir=TestConfig.WORKING_DIR,
search_params={"sources": [MemorySource.MEMORY]},
)
await reme_fs_memory.start()
result_json = await reme_fs_memory.memory_search(
query=query,
max_results=5,
sources=[MemorySource.MEMORY],
)
await reme_fs_memory.close()
import json
memory_results = json.loads(result_json)
@ -396,11 +397,18 @@ async def test_memory_search_with_source_filter():
# Search only SESSIONS source
print(f"\n--- Searching SESSIONS source for: '{query}' ---")
result_json = await reme_fs.memory_search(
# Create a new instance with SESSIONS source filter
reme_fs_sessions = ReMeFs(
enable_logo=False,
working_dir=TestConfig.WORKING_DIR,
search_params={"sources": [MemorySource.SESSIONS]},
)
await reme_fs_sessions.start()
result_json = await reme_fs_sessions.memory_search(
query=query,
max_results=5,
sources=[MemorySource.SESSIONS],
)
await reme_fs_sessions.close()
session_results = json.loads(result_json)
print(f"Found {len(session_results)} results in SESSIONS source")
for result in session_results:
@ -597,13 +605,29 @@ async def test_memory_search_hybrid_mode():
# Test with hybrid enabled
print(f"\n--- Hybrid search (enabled) for: '{query}' ---")
result_json_hybrid = await reme_fs.memory_search(
# Create instance with hybrid enabled
reme_fs_hybrid = ReMeFs(
enable_logo=False,
working_dir=TestConfig.WORKING_DIR,
default_memory_store_config={
"backend": "sqlite",
"store_name": "test_hybrid",
"embedding_model": "default",
"fts_enabled": True,
"snippet_max_chars": 700,
},
search_params={
"hybrid_enabled": True,
"hybrid_vector_weight": 0.7,
"hybrid_text_weight": 0.3,
},
)
await reme_fs_hybrid.start()
result_json_hybrid = await reme_fs_hybrid.memory_search(
query=query,
max_results=5,
hybrid_enabled=True,
hybrid_vector_weight=0.7,
hybrid_text_weight=0.3,
)
await reme_fs_hybrid.close()
import json
@ -613,11 +637,25 @@ async def test_memory_search_hybrid_mode():
# Test with hybrid disabled (vector only)
print(f"\n--- Vector-only search for: '{query}' ---")
result_json_vector = await reme_fs.memory_search(
# Create instance with hybrid disabled
reme_fs_vector = ReMeFs(
enable_logo=False,
working_dir=TestConfig.WORKING_DIR,
default_memory_store_config={
"backend": "sqlite",
"store_name": "test_hybrid",
"embedding_model": "default",
"fts_enabled": True,
"snippet_max_chars": 700,
},
search_params={"hybrid_enabled": False},
)
await reme_fs_vector.start()
result_json_vector = await reme_fs_vector.memory_search(
query=query,
max_results=5,
hybrid_enabled=False,
)
await reme_fs_vector.close()
vector_results = json.loads(result_json_vector)
print(f"Vector search found {len(vector_results)} results")
@ -632,13 +670,29 @@ async def test_memory_search_hybrid_mode():
]
for vec_weight, text_weight in weight_configs:
result_json = await reme_fs.memory_search(
# Create instance with specific weights
reme_fs_weights = ReMeFs(
enable_logo=False,
working_dir=TestConfig.WORKING_DIR,
default_memory_store_config={
"backend": "sqlite",
"store_name": "test_hybrid",
"embedding_model": "default",
"fts_enabled": True,
"snippet_max_chars": 700,
},
search_params={
"hybrid_enabled": True,
"hybrid_vector_weight": vec_weight,
"hybrid_text_weight": text_weight,
},
)
await reme_fs_weights.start()
result_json = await reme_fs_weights.memory_search(
query=query,
max_results=5,
hybrid_enabled=True,
hybrid_vector_weight=vec_weight,
hybrid_text_weight=text_weight,
)
await reme_fs_weights.close()
results = json.loads(result_json)
print(f" Vector:{vec_weight}/Text:{text_weight} -> {len(results)} results")

2
tests/test_fs_summary.py
View file

@ -155,7 +155,6 @@ async def test_summary_personal_info_storage():
result = await reme_fs.summary(
messages=messages,
version="default",
date="2023-09-01",
)
@ -191,7 +190,6 @@ async def test_summary_detailed_profile():
result = await reme_fs.summary(
messages=messages,
version="default",
date="2023-10-01",
)