scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-08-28 05:25:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

```
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled
Details

Browse source 
docs: add ReMe2 architecture design documentation

- Add comprehensive design document (reme2.md) detailing the
  three-layer architecture (L1/L2/L3) for the vault system
- Document new protocols for folder notes and memory management
- Specify interface contracts for memory_* and vault_* tools
- Outline implementation phases from current state to target

refactor: fix typo in personal retriever class

- Correct spelling error: 'retri eved_nodes' -> 'retrieved_nodes'
  in PersonalRetriever.result assignment

chore: update gitignore with vault-related patterns

- Add '/vault' to ignore vault directory
- Add '/reme-plugin' to ignore plugin files
- Add '/reme2/vault' to ignore new vault implementation
```
This commit is contained in:
huangsen 2026-05-08 16:14:42 +08:00
parent c2e43c398b
commit 514bf35050
78 changed files with 7165 additions and 1805 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -43,4 +43,6 @@ meta_memory/*
*.db
memories/*
.reme/*
/test_data
/vault
/reme-plugin
/reme2/vault

2
reme/memory/vector_based/personal/personal_retriever.py
View file

@ -106,5 +106,5 @@ class PersonalRetriever(BaseMemoryAgent):
],
)
result["retrieved_nodes"] = self.retri eved_nodes
result["retrieved_nodes"] = self.retrieved_nodes
return result

12
reme2/application.py
View file

@ -114,14 +114,14 @@ class Application(BaseComponent):
except Exception as e:
self.logger.exception(f"Failed to close component {component.__class__.__name__}: {e}")
async def run_job(self, name: str, **kwargs) -> Response:
async def run_job(self, name: str, /, **kwargs) -> Response:
"""Execute a registered job by name."""
if name not in self.context.jobs:
raise KeyError(f"Job '{name}' not found")
job = self.context.jobs[name]
return await job(**kwargs)
async def run_stream_job(self, name: str, **kwargs) -> AsyncGenerator[StreamChunk, None]:
async def run_stream_job(self, name: str, /, **kwargs) -> AsyncGenerator[StreamChunk, None]:
"""Execute a streaming job and yield chunks."""
if name not in self.context.jobs:
raise KeyError(f"Job '{name}' not found")
@ -129,10 +129,10 @@ class Application(BaseComponent):
stream_queue = asyncio.Queue()
task = asyncio.create_task(job(stream_queue=stream_queue, **kwargs))
async for chunk in execute_stream_task(
stream_queue=stream_queue,
task=task,
task_name=name,
output_format="chunk",
stream_queue=stream_queue,
task=task,
task_name=name,
output_format="chunk",
):
assert isinstance(chunk, StreamChunk)
yield chunk

8
reme2/component/__init__.py
View file

@ -2,7 +2,9 @@
from . import as_llm
from . import as_llm_formatter
from . import chunk_store
from . import as_token_counter
from . import edge_extractor
from . import file_store
from . import client
from . import embedding
from . import file_parser
@ -27,7 +29,9 @@ __all__ = [
# base components
"as_llm",
"as_llm_formatter",
"chunk_store",
"as_token_counter",
"edge_extractor",
"file_store",
"client",
"embedding",
"file_parser",

6
reme2/component/as_llm_formatter/reme_openai_chat_formatter.py
View file

@ -35,8 +35,8 @@ class ReMeOpenAIChatFormatter(OpenAIChatFormatter):
"""Extends OpenAIChatFormatter with tool result image promotion and reasoning content support."""
async def _format(
self,
msgs: list[Msg],
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
"""Format messages into OpenAI API format.
@ -108,7 +108,7 @@ class ReMeOpenAIChatFormatter(OpenAIChatFormatter):
TextBlock(
type="text",
text="<system-info>The following are the image contents from the tool "
f"result of '{block['name']}':",
f"result of '{block['name']}':",
),
*promoted_blocks,
TextBlock(type="text", text="</system-info>"),

6
reme2/component/as_token_counter/estimate_token_counter.py
View file

@ -12,9 +12,9 @@ class EstimatedTokenCounter(TokenCounterBase):
"""
def __init__(
self,
estimate_divisor: float = 4,
encoding: str = "utf-8",
self,
estimate_divisor: float = 4,
encoding: str = "utf-8",
):
"""Initialize the estimated token counter.

10
reme2/component/base_component.py
View file

@ -20,11 +20,11 @@ class BaseComponent(ABC):
component_type = ComponentEnum.BASE
def __init__(
self,
name: str | None = None,
backend: str = "",
app_context: "ApplicationContext | None" = None,
**kwargs,
self,
name: str | None = None,
backend: str = "",
app_context: "ApplicationContext | None" = None,
**kwargs,
) -> None:
self.name: str = name or self.__class__.__name__
self.backend: str = backend

70
reme2/component/base_step.py
View file

@ -8,12 +8,11 @@ from agentscope.model import ChatModelBase
from agentscope.token import TokenCounterBase
from .base_component import BaseComponent
from .chunk_store import BaseChunkStore
from .file_store import BaseFileStore
from .embedding import BaseEmbeddingModel
from .prompt_handler import PromptHandler
from .runtime_context import RuntimeContext
from ..enumeration import ComponentEnum
from ..schema.file_graph import FileGraph
class BaseStep(BaseComponent):
@ -28,12 +27,12 @@ class BaseStep(BaseComponent):
return instance
def __init__(
self,
language: str = "",
prompt_dict: dict[str, str] | None = None,
input_mapping: dict[str, str] | None = None,
output_mapping: dict[str, str] | None = None,
**kwargs,
self,
language: str = "",
prompt_dict: dict[str, str] | None = None,
input_mapping: dict[str, str] | None = None,
output_mapping: dict[str, str] | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.language = language
@ -66,6 +65,17 @@ class BaseStep(BaseComponent):
comp = self.app_context.components[key][name]
return getattr(comp, attr) if attr else comp
def _get_component_optional(self, key: ComponentEnum, name: str = "default", attr: str | None = None):
"""Like `_get_component` but returns None instead of raising when
the component / attribute is missing. For features that should
gracefully degrade when an LLM (etc.) isn't configured."""
if self.app_context is None:
return None
comp = self.app_context.components.get(key, {}).get(name)
if comp is None:
return None
return getattr(comp, attr, None) if attr else comp
@property
def as_llm(self) -> ChatModelBase:
name = self.kwargs.get("as_llm", "default")
@ -74,31 +84,45 @@ class BaseStep(BaseComponent):
@property
def as_llm_formatter(self) -> FormatterBase:
name = self.kwargs.get("as_llm_formatter", "default")
return name if isinstance(name, FormatterBase) else self._get_component(ComponentEnum.AS_LLM_FORMATTER, name,
"formatter")
return (
name
if isinstance(name, FormatterBase)
else self._get_component(
ComponentEnum.AS_LLM_FORMATTER,
name,
"formatter",
)
)
@property
def as_token_counter(self) -> TokenCounterBase:
name = self.kwargs.get("as_token_counter", "default")
return name if isinstance(name, TokenCounterBase) else self._get_component(ComponentEnum.AS_TOKEN_COUNTER, name,
"token_counter")
return (
name
if isinstance(name, TokenCounterBase)
else self._get_component(
ComponentEnum.AS_TOKEN_COUNTER,
name,
"token_counter",
)
)
@property
def chunk_store(self) -> BaseChunkStore:
name = self.kwargs.get("chunk_store", "default")
return name if isinstance(name, BaseChunkStore) else self._get_component(ComponentEnum.CHUNK_STORE, name)
@property
def file_graph(self) -> FileGraph:
name = self.kwargs.get("file_watcher", "default")
watcher = self._get_component(ComponentEnum.FILE_WATCHER, name)
return watcher.file_graph
def file_store(self) -> BaseFileStore:
name = self.kwargs.get("file_store", "default")
return name if isinstance(name, BaseFileStore) else self._get_component(ComponentEnum.FILE_STORE, name)
@property
def embedding(self) -> BaseEmbeddingModel:
name = self.kwargs.get("embedding", "default")
return name if isinstance(name, BaseEmbeddingModel) else self._get_component(ComponentEnum.EMBEDDING_MODEL,
name)
return (
name
if isinstance(name, BaseEmbeddingModel)
else self._get_component(
ComponentEnum.EMBEDDING_MODEL,
name,
)
)
def prompt_format(self, prompt_name: str, **kwargs) -> str:
return self.prompt.prompt_format(prompt_name=prompt_name, **kwargs)

17
reme2/component/chunk_store/__init__.py
View file

@ -1,17 +0,0 @@
"""Chunk store module.
Storage backends for FileChunks with vector and full-text search.
File metadata is managed by FileGraph, not by ChunkStore.
"""
from .base_chunk_store import BaseChunkStore
from .chroma_chunk_store import ChromaChunkStore
from .local_chunk_store import LocalChunkStore
from .sqlite_chunk_store import SqliteChunkStore
__all__ = [
"BaseChunkStore",
"ChromaChunkStore",
"LocalChunkStore",
"SqliteChunkStore",
]

241
reme2/component/chunk_store/base_chunk_store.py
View file

@ -1,241 +0,0 @@
"""Abstract base class for chunk storage backends."""
import re
from abc import abstractmethod
from pathlib import Path
from ..base_component import BaseComponent
from ..embedding import BaseEmbeddingModel
from ...enumeration import ComponentEnum
from ...schema import ChunkFilter, FileChunk
class BaseChunkStore(BaseComponent):
"""Abstract base class for chunk storage backends.
Handles chunk persistence and retrieval (vector / keyword / hybrid search).
File-level metadata and search filter resolution live in FileGraph; this
layer only consumes a compiled ChunkFilter (path set) for restricting search.
"""
component_type = ComponentEnum.CHUNK_STORE
def __init__(
self,
store_name: str,
db_path: str | Path,
embedding_model: str = "default",
fts_enabled: bool = True,
**kwargs,
):
super().__init__(**kwargs)
self._embedding_model_name: str = embedding_model
self.embedding_model: BaseEmbeddingModel | None = None
self.store_name: str = store_name
self.db_path: Path = Path(db_path)
self.db_path.mkdir(parents=True, exist_ok=True)
self.vector_enabled: bool = bool(embedding_model)
self.fts_enabled: bool = fts_enabled
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
raise ValueError(
f"Invalid store name '{store_name}'. Only alphanumeric characters and underscores are allowed.",
)
if not self.vector_enabled and not self.fts_enabled:
raise ValueError("At least one of embedding_model or fts_enabled must be set.")
async def _start(self):
if not self._embedding_model_name:
return
assert self.app_context is not None, "app_context must be provided"
models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
if self._embedding_model_name not in models:
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
model = models[self._embedding_model_name]
if not isinstance(model, BaseEmbeddingModel):
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
self.embedding_model = model
async def _close(self):
"""Release embedding model reference."""
self.embedding_model = None
@property
def embedding_dim(self) -> int:
"""Return the embedding dimensionality (default 1024)."""
return self.embedding_model.dimensions if self.embedding_model else 1024
def _disable_vector_search(self, reason: str = "embedding API error") -> None:
"""Disable vector search and log a warning."""
if self.vector_enabled:
self.logger.warning(f"[{self.store_name}] Disabling vector search: {reason}")
self.vector_enabled = False
async def _get_embeddings_safe(self, texts: list[str], **kwargs) -> list[list[float]] | None:
"""Get embeddings, returning None if vector search is disabled or an error occurs."""
if not self.vector_enabled:
return None
try:
assert self.embedding_model is not None, "Embedding model not initialized"
return await self.embedding_model.get_embeddings(texts, **kwargs)
except Exception as e:
self._disable_vector_search(str(e))
return None
async def get_embedding(self, query: str, **kwargs) -> list[float] | None:
"""Get embedding for a single query string."""
result = await self._get_embeddings_safe([query], **kwargs)
return result[0] if result else None
async def get_embeddings(self, queries: list[str], **kwargs) -> list[list[float]] | None:
"""Get embeddings for a batch of query strings."""
return await self._get_embeddings_safe(queries, **kwargs)
async def get_chunk_embedding(self, chunk: FileChunk, **kwargs) -> FileChunk:
"""Attach embedding to a single FileChunk."""
chunk.embedding = await self.get_embedding(chunk.text, **kwargs)
return chunk
async def get_chunk_embeddings(self, chunks: list[FileChunk], **kwargs) -> list[FileChunk]:
"""Attach embeddings to a batch of FileChunk."""
if not chunks:
return chunks
embeddings = await self.get_embeddings([c.text for c in chunks], **kwargs)
if embeddings and len(embeddings) == len(chunks):
for chunk, emb in zip(chunks, embeddings):
chunk.embedding = emb
else:
for chunk in chunks:
chunk.embedding = None
return chunks
# -- Keyword scoring utility --------------------------------------------
@staticmethod
def _score_keyword_match(query: str, text: str) -> float:
"""Score a keyword match using word-match ratio + phrase bonus."""
words = query.split()
if not words:
return 0.0
query_lower = query.lower()
words_lower = [w.lower() for w in words]
text_lower = text.lower()
n_words = len(words)
match_count = sum(1 for w in words_lower if w in text_lower)
if match_count == 0:
return 0.0
base_score = match_count / n_words
phrase_bonus = 0.2 if n_words > 1 and query_lower in text_lower else 0.0
return min(1.0, base_score + phrase_bonus)
# -- Hybrid search (concrete, delegates to abstract vector/keyword) -----
async def hybrid_search(
self,
query: str,
limit: int,
vector_weight: float = 0.7,
candidate_multiplier: float = 3.0,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Perform hybrid search combining vector and keyword results."""
assert 0.0 <= vector_weight <= 1.0
candidates = min(200, max(1, int(limit * candidate_multiplier)))
text_weight = 1.0 - vector_weight
if self.vector_enabled and self.fts_enabled:
keyword_results = await self.keyword_search(query, candidates, chunk_filter)
vector_results = await self.vector_search(query, candidates, chunk_filter)
if not keyword_results:
return vector_results[:limit]
if not vector_results:
return keyword_results[:limit]
merged = self._merge_hybrid_results(
vector_results,
keyword_results,
vector_weight,
text_weight,
)
return merged[:limit]
elif self.vector_enabled:
return await self.vector_search(query, limit, chunk_filter)
elif self.fts_enabled:
return await self.keyword_search(query, limit, chunk_filter)
return []
@staticmethod
def _merge_hybrid_results(
vector: list[FileChunk],
keyword: list[FileChunk],
vector_weight: float,
text_weight: float,
) -> list[FileChunk]:
"""Merge vector and keyword results with weighted scoring."""
merged: dict[str, FileChunk] = {}
for result in vector:
v_score = result.scores.get("vector", 0)
result.scores["score"] = v_score * vector_weight
merged[result.unique_key] = result
for result in keyword:
key = result.unique_key
k_score = result.scores.get("keyword", 0)
if key in merged:
merged[key].scores["score"] += k_score * text_weight
else:
result.scores["score"] = k_score * text_weight
merged[key] = result
results = list(merged.values())
results.sort(key=lambda r: r.score, reverse=True)
return results
# -- Filter utility -----------------------------------------------------
@staticmethod
def _apply_filter(chunks: list[FileChunk], chunk_filter: ChunkFilter | None) -> list[FileChunk]:
if chunk_filter is None or chunk_filter.resolved_paths is None:
return chunks
return [c for c in chunks if chunk_filter.match_path(c.path)]
# -- Abstract methods ---------------------------------------------------
@abstractmethod
async def clear_all(self):
"""Clear all indexed data."""
@abstractmethod
async def upsert_chunks(self, path: str, chunks: list[FileChunk]):
"""Insert or update all chunks for a file path."""
@abstractmethod
async def delete_chunks(self, path: str):
"""Delete all chunks for a file path."""
@abstractmethod
async def get_chunks(self, path: str) -> list[FileChunk]:
"""Get all chunks for a file path."""
@abstractmethod
async def vector_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Perform vector similarity search, optionally restricted by chunk_filter."""
@abstractmethod
async def keyword_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Perform full-text/keyword search, optionally restricted by chunk_filter."""

226
reme2/component/chunk_store/chroma_chunk_store.py
View file

@ -1,226 +0,0 @@
"""ChromaDB chunk storage backend."""
import time
from .base_chunk_store import BaseChunkStore
from ..component_registry import R
from ...schema import ChunkFilter, FileChunk
try:
import chromadb
from chromadb.config import Settings
_CHROMADB_IMPORT_ERROR: Exception | None = None
except Exception as e:
_CHROMADB_IMPORT_ERROR = e
chromadb = None
Settings = None
@R.register("chroma")
class ChromaChunkStore(BaseChunkStore):
"""ChromaDB chunk storage with vector and full-text search.
Uses ChromaDB's native vector search and `where_document` $contains
for keyword matching.
"""
def __init__(self, **kwargs):
if _CHROMADB_IMPORT_ERROR is not None:
raise _CHROMADB_IMPORT_ERROR
super().__init__(**kwargs)
self.client: "chromadb.ClientAPI | None" = None
self.chunks_collection: "chromadb.Collection | None" = None
@property
def collection_name(self) -> str:
return f"chunks_{self.store_name}"
# -- Lifecycle ----------------------------------------------------------
async def _start(self, app_context=None) -> None:
self.client = chromadb.PersistentClient(
path=str(self.db_path),
settings=Settings(allow_reset=True, anonymized_telemetry=False),
)
self.chunks_collection = self.client.get_or_create_collection(
name=self.collection_name,
metadata={"hnsw:space": "cosine"},
)
self.logger.info(f"ChromaChunkStore '{self.store_name}' ready: collection={self.collection_name}")
await super()._start(app_context)
async def _close(self) -> None:
self.client = None
self.chunks_collection = None
await super()._close()
# -- Filter helper ------------------------------------------------------
@staticmethod
def _path_where(chunk_filter: ChunkFilter | None) -> dict | None:
if chunk_filter is None or chunk_filter.resolved_paths is None:
return None
paths = chunk_filter.resolved_paths
if not paths:
return {"path": "__nonexistent__"}
if len(paths) == 1:
return {"path": next(iter(paths))}
return {"path": {"$in": list(paths)}}
# -- Write operations ---------------------------------------------------
async def upsert_chunks(self, path: str, chunks: list[FileChunk]) -> None:
await self.delete_chunks(path)
if not chunks:
return
chunks = await self.get_chunk_embeddings(chunks)
ids, documents, embeddings, metadatas = [], [], [], []
now = int(time.time() * 1000)
for chunk in chunks:
ids.append(chunk.id)
documents.append(chunk.text)
embeddings.append(chunk.embedding if chunk.embedding else [0.0] * self.embedding_dim)
metadatas.append({
"path": path,
"start_line": chunk.start_line,
"end_line": chunk.end_line,
"hash": chunk.hash,
"updated_at": now,
})
self.chunks_collection.upsert(
ids=ids,
documents=documents,
embeddings=embeddings,
metadatas=metadatas,
)
async def delete_chunks(self, path: str) -> None:
results = self.chunks_collection.get(where={"path": path}, include=[])
if results["ids"]:
self.chunks_collection.delete(ids=results["ids"])
# -- Read operations ----------------------------------------------------
async def get_chunks(self, path: str) -> list[FileChunk]:
results = self.chunks_collection.get(where={"path": path}, include=["documents", "metadatas"])
chunks: list[FileChunk] = []
for cid, md, text in zip(results["ids"], results["metadatas"], results["documents"]):
chunks.append(self._chunk_from_chroma(cid, md, text))
chunks.sort(key=lambda c: c.start_line)
return chunks
# -- Search helpers -----------------------------------------------------
@staticmethod
def _chunk_from_chroma(chunk_id: str, md: dict, text: str, embedding=None) -> FileChunk:
return FileChunk(
id=chunk_id,
path=md["path"],
start_line=md["start_line"],
end_line=md["end_line"],
text=text,
hash=md["hash"],
embedding=embedding,
)
# -- Search operations --------------------------------------------------
async def vector_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
if not self.vector_enabled or not query:
return []
query_embedding = await self.get_embedding(query)
if not query_embedding:
return []
try:
results = self.chunks_collection.query(
query_embeddings=[query_embedding],
n_results=limit,
where=self._path_where(chunk_filter),
include=["documents", "metadatas", "distances"],
)
except Exception as e:
self.logger.error(f"Vector search failed: {e}")
return []
chunks = []
if results["ids"] and results["ids"][0]:
for i, cid in enumerate(results["ids"][0]):
md = results["metadatas"][0][i]
distance = results["distances"][0][i]
score = max(0.0, 1.0 - distance / 2.0)
chunk = self._chunk_from_chroma(cid, md, results["documents"][0][i])
chunk.scores = {"vector": score, "score": score}
chunks.append(chunk)
chunks.sort(key=lambda c: c.score, reverse=True)
return chunks[:limit]
async def keyword_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
if not self.fts_enabled or not query:
return []
words = query.split()
if not words:
return []
word_variants = set()
for word in words:
word_variants.add(word)
word_variants.add(word.lower())
word_variants.add(word.capitalize())
word_variants.add(word.upper())
variants_list = list(word_variants)
if len(variants_list) == 1:
where_document: dict = {"$contains": variants_list[0]}
else:
where_document = {"$or": [{"$contains": w} for w in variants_list]}
results = self.chunks_collection.get(
where=self._path_where(chunk_filter),
where_document=where_document,
include=["documents", "metadatas"],
)
chunks = []
for i, cid in enumerate(results["ids"]):
md = results["metadatas"][i]
text = results["documents"][i]
score = self._score_keyword_match(query, text)
if score == 0.0:
continue
chunk = self._chunk_from_chroma(cid, md, text)
chunk.scores = {"keyword": score, "score": score}
chunks.append(chunk)
chunks.sort(key=lambda c: c.score, reverse=True)
return chunks[:limit]
# -- Clear --------------------------------------------------------------
async def clear_all(self) -> None:
self.client.delete_collection(name=self.collection_name)
self.chunks_collection = self.client.get_or_create_collection(
name=self.collection_name,
metadata={"hnsw:space": "cosine"},
)
self.logger.info(f"Cleared all data from ChromaChunkStore '{self.store_name}'")

12
reme2/component/client/http_client.py
View file

@ -15,12 +15,12 @@ class HttpClient(BaseClient):
"""HTTP client for ReMe service."""
def __init__(
self,
action: str,
host: str | None = None,
port: int | None = None,
timeout: float = 30.0,
**kwargs,
self,
action: str,
host: str | None = None,
port: int | None = None,
timeout: float = 30.0,
**kwargs,
):
super().__init__(**kwargs)

6
reme2/component/component_registry.py
View file

@ -38,9 +38,9 @@ class ComponentRegistry:
return cls
def register(
self,
cls_or_name: type[T] | str,
name: str | None = None,
self,
cls_or_name: type[T] | str,
name: str | None = None,
) -> Callable[[type[T]], type[T]] | type[T]:
"""Register a component class. Supports direct and decorator modes."""
# Direct registration: R.register(MyClass, "name")

11
reme2/component/edge_extractor/__init__.py Normal file
View file

@ -0,0 +1,11 @@
"""Typed-edge extractors — `ComponentEnum.EDGE_EXTRACTOR`."""
from .base_edge_extractor import BaseEdgeExtractor
from .llm_edge_extractor import LLMEdgeExtractor
from .regex_edge_extractor import RegexEdgeExtractor
__all__ = [
"BaseEdgeExtractor",
"RegexEdgeExtractor",
"LLMEdgeExtractor",
]

49
reme2/component/edge_extractor/base_edge_extractor.py Normal file
View file

@ -0,0 +1,49 @@
"""Abstract base class for typed-edge extractors.
A `BaseEdgeExtractor` ingests a markdown file's body text + frontmatter
dict and returns a flat list of `FileEdge` objects. The concrete
implementations realize the two routes described in `structure.md`
§"图谱关系的双态抽取":
Fast route (regex) — RegexEdgeExtractor
Slow route (LLM IE) — LLMEdgeExtractor
Edges are returned with the `source` field already populated so
downstream consumers (file_store indexing, retrievers, maintainer) can
reason about provenance.
Resolution to absolute paths and dedup against already-indexed edges
are NOT this layer's responsibility — `target` stays raw, mirroring
how it appears in source.
"""
from __future__ import annotations
from abc import abstractmethod
from ..base_component import BaseComponent
from ...enumeration import ComponentEnum
from ...schema import FileEdge
class BaseEdgeExtractor(BaseComponent):
"""Pluggable typed-edge extractor (`ComponentEnum.EDGE_EXTRACTOR`)."""
component_type = ComponentEnum.EDGE_EXTRACTOR
@abstractmethod
async def extract(
self,
text: str,
metadata: dict | None = None,
path: str | None = None,
) -> list[FileEdge]:
"""Return the edges discovered in `text` and `metadata`.
Args:
text: The body text of the source file (frontmatter stripped).
metadata: Parsed YAML frontmatter dict, or None.
path: Optional absolute path of the source file. Slow-route
extractors use this for logging / source-text caching;
fast-route ignores it.
"""

391
reme2/component/edge_extractor/llm_edge_extractor.py Normal file
View file

@ -0,0 +1,391 @@
"""LLMEdgeExtractor — slow route per `structure.md` §"双态抽取".
ReAct agent over the memory store: instead of a single-shot extraction
from the file's text alone, the agent browses the vault via read-only
memory tools (`memory_get` / `memory_list` / `memory_links` /
`memory_backlinks` / `memory_resolve_wikilink`) to ground its triples
in entities that already exist. It finalizes by calling `emit_edges`
with the JSON triples it wants to commit.
Each accepted triple becomes a `FileEdge` with `source="llm"` and
`confidence` populated from the model's self-rating.
The extractor does NOT write back to the source file — that's the
caller's job (typically the parser pipeline). It also does NOT
entity-resolve targets; the raw `object` string from the model lands
in `FileEdge.target` and is matched at the file_store layer via the
usual wikilink resolution pipeline.
Inputs that exceed `max_input_chars` are truncated; long files
should be chunked by the caller before invoking this extractor.
Triples are filtered by `min_confidence` and predicate-normalized
(lowercase, non-identifier characters → underscore) when
`predicate_normalizer=True`.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
import frontmatter
from agentscope.agent import ReActAgent
from agentscope.formatter import FormatterBase
from agentscope.message import Msg, TextBlock
from agentscope.model import ChatModelBase
from agentscope.tool import Toolkit, ToolResponse
from pydantic import BaseModel, Field
from .base_edge_extractor import BaseEdgeExtractor
from ..component_registry import R
from ..file_store import BaseFileStore
from ...enumeration import ComponentEnum
from ...schema import FileEdge
_SYSTEM_PROMPT = """You are a high-precision information-extraction agent for a personal markdown vault.
Your job for one file: identify subject-predicate-object triples that capture meaningful, lasting relations between named entities, then emit them by calling `emit_edges`. Quality > recall.
You have read-only access to the vault via these tools:
- `memory_list(path_prefix, tags, limit)` — browse indexed files
- `memory_get(path)` — read frontmatter + body + edges of one file
- `memory_resolve_wikilink(wikilink)` — check whether `[[X]]` resolves to a real path
- `memory_links(path)` — outgoing edges from `path`
- `memory_backlinks(path)` — incoming edges to `path`
- `emit_edges(triples)` — finalize: list of {subject, predicate, object, confidence}
Recommended loop (a few iterations is fine; do NOT exhaust the vault):
1. Read the source file's text and metadata (provided in the user message).
2. Generate candidate object entities from the text.
3. Probe the vault: prefer `memory_resolve_wikilink` to confirm a candidate exists; fall back to `memory_list` for fuzzy browsing only when needed.
4. Drop candidates that are pronouns / generic concepts / vague references / self-references.
5. Call `emit_edges` ONCE with the final filtered set, then stop.
Triple rules:
- subject: the source file's main entity (typically the file name / first H1).
- predicate: identifier-shaped, lowercase, words joined with underscore (e.g. `works_at`, `authored`, `depends_on`, `mentions`).
- object: the target entity name as it appears in the text — do NOT wrap with `[[ ]]`.
- confidence: float in [0, 1]; the parser drops anything below `min_confidence`.
Skip silently: pronouns, self-referential relations, transient events, generic concepts.
If nothing meaningful is present, call `emit_edges([])` and stop."""
_USER_TEMPLATE = """Source file: {path}
Frontmatter:
{frontmatter}
Body:
{text}
Extract triples grounded in the vault now. Use the read tools as needed, then call `emit_edges` ONCE with the final list."""
class _Triple(BaseModel):
"""One subject-predicate-object triple from the LLM."""
subject: str = Field(description="Source entity of the relation.")
predicate: str = Field(description="Identifier-shaped relation name.")
object: str = Field(description="Target entity name.")
confidence: float = Field(ge=0.0, le=1.0, description="Self-rated confidence.")
_LINK_WRAPPER_RE = re.compile(r"^!?\[\[([^\]\|\#]+?)(?:[#\|][^\]]*)?\]\]$")
_PREDICATE_NORM_RE = re.compile(r"[^a-z0-9_]+")
def _strip_link_wrapper(s: str) -> str:
"""Strip `[[X]]` / `![[X#h|alias]]` wrappers if the LLM emitted them."""
s = s.strip().strip('"\'')
if m := _LINK_WRAPPER_RE.match(s):
return m.group(1).strip()
return s
def _normalize_predicate(pred: str) -> str:
"""Lowercase + collapse non-identifier chars to `_`."""
pred = pred.strip().lower()
pred = _PREDICATE_NORM_RE.sub("_", pred).strip("_")
return pred or "related"
def _text_response(payload: object) -> ToolResponse:
text = json.dumps(payload, ensure_ascii=False, indent=2, default=str)
return ToolResponse(content=[TextBlock(type="text", text=text)])
@R.register("llm")
class LLMEdgeExtractor(BaseEdgeExtractor):
"""LLM-driven typed-edge extractor (slow route, ReAct over memory tools)."""
def __init__(
self,
as_llm: str = "default",
as_llm_formatter: str = "default",
file_store: str = "default",
max_input_chars: int = 8000,
min_confidence: float = 0.5,
predicate_normalizer: bool = True,
max_iters: int = 8,
console_enabled: bool = False,
**kwargs,
):
super().__init__(**kwargs)
self._as_llm_name: str = as_llm
self._as_llm_formatter_name: str = as_llm_formatter
self._file_store_name: str = file_store
self.max_input_chars: int = int(max_input_chars)
self.min_confidence: float = float(min_confidence)
self.predicate_normalizer: bool = bool(predicate_normalizer)
self.max_iters: int = int(max_iters)
self.console_enabled: bool = bool(console_enabled)
self.as_llm: ChatModelBase | None = None
self.as_llm_formatter: FormatterBase | None = None
self.file_store: BaseFileStore | None = None
async def _start(self) -> None:
assert self.app_context is not None, "LLMEdgeExtractor requires app_context"
llms = self.app_context.components.get(ComponentEnum.AS_LLM, {})
wrapper = llms.get(self._as_llm_name)
if wrapper is None:
raise ValueError(f"as_llm '{self._as_llm_name}' not configured")
model = getattr(wrapper, "model", None)
if not isinstance(model, ChatModelBase):
raise TypeError(
f"as_llm '{self._as_llm_name}'.model is {type(model).__name__}, "
f"expected ChatModelBase",
)
self.as_llm = model
formatters = self.app_context.components.get(ComponentEnum.AS_LLM_FORMATTER, {})
fwrapper = formatters.get(self._as_llm_formatter_name)
formatter = getattr(fwrapper, "formatter", None) if fwrapper is not None else None
if not isinstance(formatter, FormatterBase):
raise TypeError(
f"as_llm_formatter '{self._as_llm_formatter_name}' missing or wrong type",
)
self.as_llm_formatter = formatter
stores = self.app_context.components.get(ComponentEnum.FILE_STORE, {})
store = stores.get(self._file_store_name)
if not isinstance(store, BaseFileStore):
raise ValueError(
f"file_store '{self._file_store_name}' not configured for LLMEdgeExtractor",
)
self.file_store = store
async def _close(self) -> None:
self.as_llm = None
self.as_llm_formatter = None
self.file_store = None
async def extract(
self,
text: str,
metadata: dict | None = None,
path: str | None = None,
) -> list[FileEdge]:
if not text or not text.strip():
return []
if self.as_llm is None or self.as_llm_formatter is None or self.file_store is None:
raise RuntimeError("LLMEdgeExtractor not started; call .start() first")
body = text if len(text) <= self.max_input_chars else text[: self.max_input_chars]
toolkit, sink = self._build_toolkit()
agent = ReActAgent(
name="llm_edge_extractor",
model=self.as_llm,
sys_prompt=_SYSTEM_PROMPT,
formatter=self.as_llm_formatter,
toolkit=toolkit,
max_iters=self.max_iters,
)
agent.set_console_output_enabled(self.console_enabled)
user_msg = _USER_TEMPLATE.format(
path=path or "(unknown)",
frontmatter=json.dumps(metadata or {}, ensure_ascii=False, indent=2, default=str),
text=body,
)
try:
await agent.reply(Msg(name="user", role="user", content=user_msg))
except Exception as e:
self.logger.warning(f"LLM IE agent failed for {path}: {e}")
return []
return self._sink_to_edges(sink)
# -- Tool plumbing ------------------------------------------------------
def _build_toolkit(self) -> tuple[Toolkit, list[_Triple]]:
"""Build a read-only memory toolkit + an `emit_edges` sink.
Returns `(toolkit, sink)`. The sink is the mutable list the
`emit_edges` tool appends parsed triples into.
"""
assert self.file_store is not None
store = self.file_store
vault_root = store.vault_root or Path.cwd().resolve()
sink: list[_Triple] = []
def _resolve(p: str) -> Path:
pp = Path(p)
if not pp.is_absolute():
pp = vault_root / pp
return pp.resolve()
async def memory_get(path: str) -> ToolResponse:
"""Read one vault file: frontmatter + body + outgoing edges.
Args:
path (str): Absolute or vault-relative path to the file.
"""
target = _resolve(path)
out: dict = {"path": str(target), "exists": False}
meta = store.get_file_meta(str(target))
if meta is not None:
edges = store.get_edges(str(target))
out.update({
"exists": True,
"metadata": meta.metadata,
"edges": [e.model_dump(exclude_none=True) for e in edges],
})
if target.is_file():
try:
raw = target.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
out["exists"] = True
out["metadata"] = dict(post.metadata)
out["content"] = post.content
except Exception as e:
out["read_error"] = str(e)
return _text_response(out)
async def memory_list(
path_prefix: str = "",
tags: list[str] | None = None,
limit: int = 50,
) -> ToolResponse:
"""List indexed vault files, filtered by path prefix and tags.
Args:
path_prefix (str): Restrict to paths starting with this prefix.
tags (list[str] | None): All tags must be present on a file.
limit (int): Cap on returned items.
"""
items: list[dict] = []
for p, meta in store.nodes.items():
if path_prefix and not p.startswith(path_prefix):
continue
md = meta.metadata or {}
if tags:
file_tags = set(md.get("tags", []) or [])
if not all(t in file_tags for t in tags):
continue
items.append({"path": p, "file": meta.file, "metadata": md})
if len(items) >= limit:
break
return _text_response({"items": items, "count": len(items)})
async def memory_resolve_wikilink(wikilink: str) -> ToolResponse:
"""Resolve a `[[wikilink]]` to its absolute path (or null if dangling).
Args:
wikilink (str): The bare wikilink target (no brackets).
"""
hit = store.resolve_wikilink(wikilink)
return _text_response({"wikilink": wikilink, "path": hit})
async def memory_links(path: str) -> ToolResponse:
"""Outgoing edges from `path` (resolved targets only).
Args:
path (str): Absolute or vault-relative path.
"""
target = _resolve(path)
out = [
{"path": m.path, "predicate": e.predicate, "metadata": m.metadata}
for m, e in store.get_links(str(target))
]
return _text_response({"path": str(target), "links": out})
async def memory_backlinks(path: str) -> ToolResponse:
"""Incoming edges to `path` (files that link TO it).
Args:
path (str): Absolute or vault-relative path.
"""
target = _resolve(path)
out = [
{"path": m.path, "predicate": e.predicate, "metadata": m.metadata}
for m, e in store.get_backlinks(str(target))
]
return _text_response({"path": str(target), "backlinks": out})
async def emit_edges(triples: list[dict]) -> ToolResponse:
"""Finalize extraction. Emit the full set of triples for this file.
Call this exactly ONCE per file, with all triples you want to
commit. After this call, stop and produce a brief textual reply
summarizing what you emitted.
Args:
triples (list[dict]): List of {subject, predicate, object, confidence}.
`confidence` ∈ [0, 1]; values below the configured threshold
are dropped by the parser.
"""
accepted = 0
rejected = 0
for t in triples:
try:
sink.append(_Triple.model_validate(t))
accepted += 1
except Exception:
rejected += 1
return _text_response({
"accepted": accepted,
"rejected": rejected,
"total_so_far": len(sink),
})
toolkit = Toolkit()
for fn in (
memory_get,
memory_list,
memory_resolve_wikilink,
memory_links,
memory_backlinks,
emit_edges,
):
toolkit.register_tool_function(fn, namesake_strategy="override")
return toolkit, sink
def _sink_to_edges(self, sink: list[_Triple]) -> list[FileEdge]:
edges: list[FileEdge] = []
for tri in sink:
if tri.confidence < self.min_confidence:
continue
target = _strip_link_wrapper(tri.object)
if not target or target == _strip_link_wrapper(tri.subject):
continue
predicate = (
_normalize_predicate(tri.predicate)
if self.predicate_normalizer
else tri.predicate.strip()
)
edges.append(FileEdge(
target=target,
predicate=predicate or None,
source="llm",
confidence=tri.confidence,
))
return edges

47
reme2/component/edge_extractor/regex_edge_extractor.py Normal file
View file

@ -0,0 +1,47 @@
"""RegexEdgeExtractor — fast route per `structure.md` §"双态抽取".
Pulls edges from inline syntax + frontmatter walk in O(text length).
Synchronous and dependency-free; runs inside the watcher's hot path
without blocking on any external service.
Sources of edges:
- body inline syntax: `[[X]]`, `![[X#Section|Alias]]`,
`[predicate:: [[X]]]`, `(predicate:: [[X]])`
- frontmatter walk: `author: "[[John]]"` → predicate=author,
`related: ["[[X]]", "[[Y]]"]` → two edges
Dedup key: (target, predicate, anchor, alias, embed). When the same
edge appears in both body and frontmatter, the inline (regex) variant
wins — it's the more direct authoring intent.
"""
from __future__ import annotations
from .base_edge_extractor import BaseEdgeExtractor
from ..component_registry import R
from ...schema import FileEdge
from ...utils.wikilink import parse_wikilinks, parse_wikilinks_from_metadata
@R.register("regex")
class RegexEdgeExtractor(BaseEdgeExtractor):
"""Inline-syntax + frontmatter typed-edge extractor."""
async def extract(
self,
text: str,
metadata: dict | None = None,
path: str | None = None,
) -> list[FileEdge]:
body_edges = parse_wikilinks(text or "")
meta_edges = parse_wikilinks_from_metadata(metadata or {})
seen: set[tuple] = set()
out: list[FileEdge] = []
for edge in (*body_edges, *meta_edges):
key = (edge.target, edge.predicate, edge.anchor, edge.alias, edge.embed)
if key in seen:
continue
seen.add(key)
out.append(edge)
return out

32
reme2/component/embedding/base_embedding_model.py
View file

@ -26,17 +26,17 @@ class BaseEmbeddingModel(BaseComponent):
component_type = ComponentEnum.EMBEDDING_MODEL
def __init__(
self,
api_key: str | None = None,
base_url: str | None = None,
model_name: str = "",
dimensions: int = 1024,
pass_dimensions: bool = False,
max_batch_size: int = 10,
max_input_length: int = 8192,
max_cache_size: int = 2000,
enable_cache: bool = True,
**kwargs,
self,
api_key: str | None = None,
base_url: str | None = None,
model_name: str = "",
dimensions: int = 1024,
pass_dimensions: bool = False,
max_batch_size: int = 10,
max_input_length: int = 8192,
max_cache_size: int = 2000,
enable_cache: bool = True,
**kwargs,
):
super().__init__(**kwargs)
self.api_key: str = api_key or os.environ.get("EMBEDDING_API_KEY", "")
@ -81,7 +81,7 @@ class BaseEmbeddingModel(BaseComponent):
self.logger.warning(f"Embedding dim {actual_len} < expected {self.dimensions}, padding")
return embedding + [0.0] * (self.dimensions - actual_len)
self.logger.warning(f"Embedding dim {actual_len} > expected {self.dimensions}, truncating")
return embedding[:self.dimensions]
return embedding[: self.dimensions]
def _get_cache_key(self, text: str) -> str:
"""Generate cache key from text + model_name + dimensions."""
@ -109,7 +109,9 @@ class BaseEmbeddingModel(BaseComponent):
for key, emb in zip(data["keys"], data["embeddings"]):
emb_list = emb.tolist()
if len(emb_list) != self.dimensions:
self.logger.warning(f"Cache dimension mismatch for {key}: expected {self.dimensions}, got {len(emb_list)}")
self.logger.warning(
f"Cache dimension mismatch for {key}: expected {self.dimensions}, got {len(emb_list)}",
)
continue
if len(self._embedding_cache) >= self.max_cache_size:
self.logger.info(f"Cache limit reached ({self.max_cache_size}), loaded {loaded_count}")
@ -188,7 +190,7 @@ class BaseEmbeddingModel(BaseComponent):
async def get_embeddings(self, input_text: list[str], **kwargs) -> list[list[float] | None]:
"""Get embeddings for multiple texts with cache and batching."""
# TODO change to bytes instead of str
truncated_texts = [t[:self.max_input_length] for t in input_text]
truncated_texts = [t[: self.max_input_length] for t in input_text]
results: list[list[float] | None] = [None] * len(truncated_texts)
texts_to_compute: list[tuple[int, str]] = []
@ -202,7 +204,7 @@ class BaseEmbeddingModel(BaseComponent):
if texts_to_compute:
uncached_texts = [text for _, text in texts_to_compute]
for i in range(0, len(uncached_texts), self.max_batch_size):
batch = texts_to_compute[i:i + self.max_batch_size]
batch = texts_to_compute[i : i + self.max_batch_size]
batch_indices = [idx for idx, _ in batch]
batch_texts = [text for _, text in batch]

6
reme2/component/file_parser/__init__.py
View file

@ -1,11 +1,15 @@
"""File parser implementations for different file formats."""
from .base_file_parser import BaseFileParser
from .default_file_parser import DefaultFileParser
from .md_file_parser import MdFileParser
from .text_file_parser import TextFileParser
# Backwards-compat alias.
DefaultFileParser = TextFileParser
__all__ = [
"BaseFileParser",
"DefaultFileParser",
"MdFileParser",
"TextFileParser",
]

90
reme2/component/file_parser/base_file_parser.py
View file

@ -1,18 +1,98 @@
"""Abstract base class for file parsers."""
"""Abstract base class for file parsers.
Single-pass parser: `parse(path, existing_chunks)` returns a `ParsedFile`
carrying metadata + chunks (with embeddings attached) + edges.
The parser owns the embedding pipeline. To avoid re-embedding unchanged
blocks, the watcher passes in the file's prior chunks; the parser
hash-diffs and only calls the embedding API for blocks whose hash is
new. Vanished hashes simply drop out (the file_store's upsert is a
delete-and-insert).
If `existing_chunks` is None / empty, every chunk is embedded fresh.
"""
from abc import abstractmethod
from ..base_component import BaseComponent
from ..embedding import BaseEmbeddingModel
from ...enumeration import ComponentEnum, FileSuffixEnum
from ...schema import FileChunk, FileMetadata
from ...schema import FileChunk, ParsedFile
class BaseFileParser(BaseComponent):
"""Parser that declares handled suffixes and produces FileChunks."""
"""Single-pass parser producing an embedded `ParsedFile`."""
component_type = ComponentEnum.FILE_PARSER
suffixes: list[FileSuffixEnum] = []
def __init__(self, embedding_model: str = "", **kwargs):
super().__init__(**kwargs)
self._embedding_model_name: str = embedding_model
self.embedding_model: BaseEmbeddingModel | None = None
async def _start(self) -> None:
if not self._embedding_model_name:
return
assert self.app_context is not None, "app_context must be provided"
models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
if self._embedding_model_name not in models:
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
model = models[self._embedding_model_name]
if not isinstance(model, BaseEmbeddingModel):
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
self.embedding_model = model
async def _close(self) -> None:
self.embedding_model = None
async def _embed_chunks(self, chunks: list[FileChunk]) -> list[FileChunk]:
"""Attach embeddings to chunks if an embedding model is configured.
Failures (missing embeddings) leave `chunk.embedding=None`; the
file_store can still keyword-search them.
"""
if not chunks or self.embedding_model is None:
return chunks
try:
await self.embedding_model.get_node_embeddings(chunks)
except Exception as e:
self.logger.warning(f"embedding chunks failed: {e}")
return chunks
@staticmethod
def _hash_diff_attach(
chunks: list[FileChunk],
existing_chunks: list[FileChunk] | None,
) -> list[FileChunk]:
"""Attach cached embeddings to chunks whose hash already exists.
Returns the dirty subset (chunks still needing embeddings). Mutates
the input chunks in place: matched chunks get their `embedding`
field set from the cache.
"""
if not existing_chunks:
return list(chunks)
cached = {c.hash: c.embedding for c in existing_chunks if c.embedding}
if not cached:
return list(chunks)
dirty: list[FileChunk] = []
for c in chunks:
cached_emb = cached.get(c.hash)
if cached_emb is not None:
c.embedding = cached_emb
else:
dirty.append(c)
return dirty
@abstractmethod
async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
"""Parse a file into metadata and chunks."""
async def parse(
self,
path: str,
existing_chunks: list[FileChunk] | None = None,
) -> ParsedFile:
"""Parse + embed the file, returning a fully populated `ParsedFile`.
`existing_chunks`, when provided, lets the parser reuse cached
embeddings for blocks whose hash hasn't changed.
"""

230
reme2/component/file_parser/md_file_parser.py
View file

@ -1,54 +1,226 @@
"""Markdown file parser."""
"""Markdown file parser — frontmatter + wikilink graph + AST-aware chunking.
import asyncio
The chunker splits markdown into semantic blocks using:
- ATX headings (#, ##, ..., ######) as section anchors
- Blank lines (paragraph boundaries) as soft splits within a section
- Code fences (``` or ~~~) preserved as a single block
Each block carries a `heading_path` breadcrumb prepended to its text — gives
the embedding model section context AND lets retrieval results show callers
where the hit lives. The hash is computed over the final text (with
breadcrumb), so renaming a heading correctly invalidates child block
embeddings.
Hash-diff cache compatibility: blocks with identical (heading_path + body)
across edits produce the same hash, so the watcher can reuse old embeddings
and only call the embedding API for dirty blocks.
Edge extraction is delegated to a `BaseEdgeExtractor` resolved from the
app context (`edge_extractor` constructor arg, defaults to "default"). If
the app context isn't bound or the named extractor isn't registered, the
parser falls back to a local `RegexEdgeExtractor` so unit tests / simple
scripts can still parse without YAML wiring.
"""
import re
from pathlib import Path
import frontmatter
from .base_file_parser import BaseFileParser
from ..component_registry import R
from ...enumeration import FileSuffixEnum
from ...schema import FileChunk, FileMetadata
from ...utils import chunk_markdown
from ..edge_extractor import BaseEdgeExtractor, RegexEdgeExtractor
from ...enumeration import ComponentEnum, FileSuffixEnum
from ...schema import FileChunk, ParsedFile
from ...utils import hash_text
@R.register("md")
class MdFileParser(BaseFileParser):
"""Parser for Markdown files with YAML frontmatter support."""
"""Parser for Markdown files with YAML frontmatter and wikilink support."""
suffixes = [FileSuffixEnum.MD, FileSuffixEnum.MARKDOWN]
def __init__(self, encoding: str = "utf-8", chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs):
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$")
_FENCE_RE = re.compile(r"^(```|~~~)")
def __init__(
self,
encoding: str = "utf-8",
edge_extractor: str = "default",
**kwargs,
):
super().__init__(**kwargs)
self.encoding = encoding
self.chunk_tokens = chunk_tokens
self.chunk_overlap = chunk_overlap
self._edge_extractor_name: str = edge_extractor
self.edge_extractor: BaseEdgeExtractor | None = None
async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
async def _start(self) -> None:
await super()._start()
# Try to resolve the configured extractor; fall back to a local
# regex extractor when no app_context / no matching component.
resolved: BaseEdgeExtractor | None = None
if self.app_context is not None and self._edge_extractor_name:
extractors = self.app_context.components.get(ComponentEnum.EDGE_EXTRACTOR, {})
candidate = extractors.get(self._edge_extractor_name)
if isinstance(candidate, BaseEdgeExtractor):
resolved = candidate
if resolved is None:
resolved = RegexEdgeExtractor()
await resolved.start()
self.edge_extractor = resolved
async def _close(self) -> None:
await super()._close()
self.edge_extractor = None
async def parse(
self,
path: str,
existing_chunks: list[FileChunk] | None = None,
) -> ParsedFile:
file_path = Path(path)
raw = file_path.read_text(encoding=self.encoding)
post = frontmatter.loads(raw)
stat = file_path.stat()
metadata = dict(post.metadata)
content = post.content
absolute_path = str(file_path.absolute())
def _read_and_parse():
raw = file_path.read_text(encoding=self.encoding)
post = frontmatter.loads(raw)
stat = file_path.stat()
return stat, dict(post.metadata), post.content
extractor = self.edge_extractor or RegexEdgeExtractor()
edges = await extractor.extract(content, metadata, path=absolute_path)
stat, metadata, content = await asyncio.to_thread(_read_and_parse)
chunks = self.chunk_markdown(content, absolute_path)
dirty = self._hash_diff_attach(chunks, existing_chunks)
if dirty:
await self._embed_chunks(dirty)
file_meta = FileMetadata(
modified_time=stat.st_mtime,
path=str(file_path.absolute()),
return ParsedFile(
file=file_path.stem,
path=absolute_path,
st_mtime=stat.st_mtime,
metadata=metadata,
edges=edges,
chunks=chunks,
)
chunks = (
chunk_markdown(
content,
file_meta.path,
self.chunk_tokens,
self.chunk_overlap,
)
or []
)
# -- Chunker ----------------------------------------------------------
return file_meta, chunks
@staticmethod
def _breadcrumb(heading_path: list[str]) -> str:
return " > ".join(heading_path) if heading_path else ""
@classmethod
def _make_block_text(cls, heading_path: list[str], body: str) -> str:
"""Compose final block text: breadcrumb line (if any) + blank + body."""
body = body.rstrip("\n")
crumb = cls._breadcrumb(heading_path)
if crumb:
return f"{crumb}\n\n{body}" if body else crumb
return body
@classmethod
def chunk_markdown(cls, text: str, path: str) -> list[FileChunk]:
"""Split markdown into AST-aware blocks (headings / paragraphs / fences)."""
if not text or not text.strip():
return []
lines = text.split("\n")
chunks: list[FileChunk] = []
heading_stack: list[tuple[int, str]] = [] # [(level, title)]
body_lines: list[str] = []
body_start = 1
in_fence = False
fence_marker = ""
def current_path() -> list[str]:
return [t for _, t in heading_stack]
def emit(block_text: str, start_line: int, end_line: int) -> None:
h = hash_text(block_text)
chunks.append(
FileChunk(
id=hash_text(f"{path}::{start_line}::{end_line}::{h}::{len(chunks)}"),
path=path,
start_line=start_line,
end_line=end_line,
text=block_text,
hash=h,
),
)
def flush_body(end_line: int) -> None:
nonlocal body_lines, body_start
if not body_lines:
return
# Strip leading/trailing blank lines from the block (paragraph
# boundaries eat their own newline, but whitespace can sneak in
# via the fence path).
while body_lines and not body_lines[0].strip():
body_lines.pop(0)
body_start += 1
while body_lines and not body_lines[-1].strip():
body_lines.pop()
end_line -= 1
if not body_lines:
body_lines = []
return
raw_body = "\n".join(body_lines)
block_text = cls._make_block_text(current_path(), raw_body)
emit(block_text, body_start, end_line)
body_lines = []
for i, line in enumerate(lines, 1):
stripped = line.strip()
# Code fences: keep contents intact, no inner splits.
if not in_fence and cls._FENCE_RE.match(stripped):
if body_lines:
flush_body(end_line=i - 1)
in_fence = True
fence_marker = stripped[:3]
body_lines = [line]
body_start = i
continue
if in_fence:
body_lines.append(line)
if stripped.startswith(fence_marker):
in_fence = False
fence_marker = ""
flush_body(end_line=i)
continue
# ATX heading: closes prior block, opens a new section.
m = cls._HEADING_RE.match(line)
if m:
if body_lines:
flush_body(end_line=i - 1)
level = len(m.group(1))
title = m.group(2).strip()
heading_stack = [(lv, t) for lv, t in heading_stack if lv < level]
heading_stack.append((level, title))
# Heading line itself becomes a block (so the heading text is
# searchable as its own unit).
block_text = cls._make_block_text(current_path(), "")
emit(block_text, i, i)
body_start = i + 1
continue
# Blank line: paragraph boundary.
if not stripped:
if body_lines:
flush_body(end_line=i - 1)
body_start = i + 1
continue
# Regular content line.
if not body_lines:
body_start = i
body_lines.append(line)
if body_lines:
flush_body(end_line=len(lines))
return chunks

44
reme2/component/file_parser/text_file_parser.py
View file

@ -8,7 +8,7 @@ import aiofiles
from .base_file_parser import BaseFileParser
from ..component_registry import R
from ...enumeration import FileSuffixEnum
from ...schema import FileChunk, FileMetadata
from ...schema import FileChunk, ParsedFile
@R.register("text")
@ -23,9 +23,14 @@ class TextFileParser(BaseFileParser):
self.chunk_size = max(32, chunk_tokens * 4)
self.overlap_size = max(0, chunk_overlap * 4)
async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
async def parse(
self,
path: str,
existing_chunks: list[FileChunk] | None = None,
) -> ParsedFile:
file_path = Path(path)
stat = file_path.stat()
absolute_path = str(file_path.absolute())
try:
async with aiofiles.open(file_path, encoding=self.encoding) as f:
@ -34,17 +39,20 @@ class TextFileParser(BaseFileParser):
async with aiofiles.open(file_path, encoding=self.encoding, errors="ignore") as f:
content = await f.read()
except Exception:
content = None
content = ""
file_meta = FileMetadata(
chunks = self._chunk(content, absolute_path) if content else []
dirty = self._hash_diff_attach(chunks, existing_chunks)
if dirty:
await self._embed_chunks(dirty)
return ParsedFile(
file=file_path.stem,
path=str(file_path.absolute()),
path=absolute_path,
st_mtime=stat.st_mtime,
chunks=chunks,
)
chunks = self._chunk(content, file_meta.path) if content else []
return file_meta, chunks
def _chunk(self, text: str, path: str) -> list[FileChunk]:
"""Split text into chunks with overlap."""
if not text.strip():
@ -58,7 +66,7 @@ class TextFileParser(BaseFileParser):
for line_no, line in enumerate(lines, 1):
# Split long lines into segments
for start in range(0, max(1, len(line)), self.chunk_size):
seg = line[start:start + self.chunk_size]
seg = line[start : start + self.chunk_size]
seg_chars = len(seg) + 1 # +1 for newline
# Flush when buffer would exceed limit
@ -82,14 +90,16 @@ class TextFileParser(BaseFileParser):
h = hashlib.sha256(chunk_text.encode()).hexdigest()
chunk_id = hashlib.sha256(f"{path}:{start_line}:{end_line}:{h}:{len(chunks)}".encode()).hexdigest()
chunks.append(FileChunk(
id=chunk_id,
path=path,
start_line=start_line,
end_line=end_line,
text=chunk_text,
hash=h,
))
chunks.append(
FileChunk(
id=chunk_id,
path=path,
start_line=start_line,
end_line=end_line,
text=chunk_text,
hash=h,
),
)
def _carry_overlap(self, buf: list[tuple[str, int]]) -> tuple[list[tuple[str, int]], int]:
"""Keep overlapping lines from the end of buffer."""

26
reme2/component/file_store/__init__.py Normal file
View file

@ -0,0 +1,26 @@
"""File store module.
Unified storage for file metadata (frontmatter + mtime), the wikilink
graph (edges per path), and chunks (text + embeddings) with vector /
keyword / hybrid search.
The store accepts a `ParsedFile` from the watcher's parsing pass via
`upsert_parsed(...)`, dispatching meta + edges + chunks into the
appropriate persistence pipelines.
Two backends:
LocalFileStore — pure-Python with JSONL persistence (default,
zero deps, fine for small vaults).
SqliteFileStore — SQLite + FTS5 + sqlite-vec for keyword/vector
at scale; meta + edges live in relational tables.
"""
from .base_file_store import BaseFileStore
from .local_file_store import LocalFileStore
from .sqlite_file_store import SqliteFileStore
__all__ = [
"BaseFileStore",
"LocalFileStore",
"SqliteFileStore",
]

629
reme2/component/file_store/base_file_store.py Normal file
View file

@ -0,0 +1,629 @@
"""Abstract base class for file stores.
A `FileStore` manages **everything** about a file in the vault:
- file metadata (frontmatter + mtime)
- the wikilink graph: edges per file (links / backlinks / resolution)
- graph-walk retrieval primitives (BFS expansion, decay scoring)
- chunks (text + embeddings + position) with vector / keyword / hybrid search
Edges are stored independently of metadata in `_edges: dict[path,
list[FileEdge]]`. The watcher submits a `ParsedFile` via
`upsert_parsed(parsed)`; the store fans this out to meta + edge + chunk
persistence in a single atomic-from-the-caller's-POV operation.
Reads (`get_file_meta`, `get_links`, `resolve_wikilink`, `subgraph_score`,
`filter`, ...) are **synchronous** — they hit an in-memory index that is
rebuilt from the persistent backend on `_start`. Writes are **async** —
they touch persistence first, then update the in-memory index, so on
crash the on-disk view is the source of truth.
Backends supply persistence for chunks, file metadata, AND edges via
abstract methods. The default meta + edge persistence is sidecar JSONL;
SQLite overrides with relational tables.
"""
import re
from abc import abstractmethod
from collections import defaultdict, deque
from pathlib import Path
from typing import Iterable
from ..base_component import BaseComponent
from ..embedding import BaseEmbeddingModel
from ...enumeration import ComponentEnum
from ...schema import ChunkFilter, FileChunk, FileEdge, FileMetadata, ParsedFile
from ...utils.wikilink import extract_wikilinks
def _target_stem(raw: str) -> str:
"""Stem of a raw wikilink target — `"topics/X/X"` → `"X"`."""
target = raw.strip()
if target.endswith(".md"):
target = target[:-3]
return target.rsplit("/", 1)[-1]
class BaseFileStore(BaseComponent):
"""Unified file store: metadata + edges + chunks + graph + search."""
component_type = ComponentEnum.FILE_STORE
def __init__(
self,
store_name: str,
db_path: str | Path,
embedding_model: str = "default",
fts_enabled: bool = True,
**kwargs,
):
super().__init__(**kwargs)
self._embedding_model_name: str = embedding_model
self.embedding_model: BaseEmbeddingModel | None = None
self.store_name: str = store_name
self.db_path: Path = Path(db_path)
self.db_path.mkdir(parents=True, exist_ok=True)
self.vector_enabled: bool = bool(embedding_model)
self.fts_enabled: bool = fts_enabled
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
raise ValueError(
f"Invalid store name '{store_name}'. Only alphanumeric characters and underscores are allowed.",
)
if not self.vector_enabled and not self.fts_enabled:
raise ValueError("At least one of embedding_model or fts_enabled must be set.")
# In-memory indices (rebuilt on _start from persisted state).
self._nodes: dict[str, FileMetadata] = {}
self._edges: dict[str, list[FileEdge]] = {}
self._stems: dict[str, set[str]] = defaultdict(set)
self._backlinks: dict[str, set[str]] = defaultdict(set)
self.vault_root: Path | None = None
# -- Lifecycle ----------------------------------------------------------
async def _start(self):
if self._embedding_model_name:
assert self.app_context is not None, "app_context must be provided"
models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
if self._embedding_model_name not in models:
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
model = models[self._embedding_model_name]
if not isinstance(model, BaseEmbeddingModel):
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
self.embedding_model = model
# Concrete backend's _start runs first (opens conn, creates tables);
# this base _start runs LAST so persistence is ready by the time we
# iterate it. Subclasses should `await super()._start()` at the END
# of their own _start.
await self._reload_file_metas()
async def _close(self):
self.embedding_model = None
self._nodes.clear()
self._edges.clear()
self._stems.clear()
self._backlinks.clear()
async def _reload_file_metas(self) -> None:
"""Rebuild in-memory indices from persisted meta + edges. Called on _start."""
self._nodes.clear()
self._edges.clear()
self._stems.clear()
self._backlinks.clear()
count = 0
for meta in self._iter_persisted_metas():
self._nodes[meta.path] = meta
self._stems[Path(meta.path).stem].add(meta.path)
count += 1
edge_count = 0
for path, edges in self._iter_persisted_edges():
if path not in self._nodes:
continue # orphan — meta was already pruned
self._edges[path] = edges
for edge in edges:
self._backlinks[_target_stem(edge.target)].add(path)
edge_count += 1
self.logger.info(
f"[{self.store_name}] Loaded {count} file metas, {edge_count} edges into in-memory index",
)
# -- Embedding helpers --------------------------------------------------
@property
def embedding_dim(self) -> int:
return self.embedding_model.dimensions if self.embedding_model else 1024
def _disable_vector_search(self, reason: str = "embedding API error") -> None:
if self.vector_enabled:
self.logger.warning(f"[{self.store_name}] Disabling vector search: {reason}")
self.vector_enabled = False
async def _get_embeddings_safe(self, texts: list[str], **kwargs) -> list[list[float]] | None:
if not self.vector_enabled:
return None
try:
assert self.embedding_model is not None
return await self.embedding_model.get_embeddings(texts, **kwargs)
except Exception as e:
self._disable_vector_search(str(e))
return None
async def get_embedding(self, query: str, **kwargs) -> list[float] | None:
result = await self._get_embeddings_safe([query], **kwargs)
return result[0] if result else None
async def get_embeddings(self, queries: list[str], **kwargs) -> list[list[float]] | None:
return await self._get_embeddings_safe(queries, **kwargs)
# -- Vault root ---------------------------------------------------------
def set_vault_root(self, vault_root: str | Path | None) -> None:
"""Bind the runtime vault root for explicit-path wikilink resolution."""
self.vault_root = Path(vault_root).resolve() if vault_root is not None else None
# -- File-meta CRUD (async writes; sync reads on in-memory index) -------
async def upsert_file_meta(self, meta: FileMetadata) -> None:
"""Persist + index a file meta. Replaces any prior entry for the path.
Edges are NOT touched here — call `upsert_edges` separately, or
use `upsert_parsed` for the combined fan-out from a `ParsedFile`.
"""
await self._persist_upsert_meta(meta)
self._nodes[meta.path] = meta
self._stems[Path(meta.path).stem].add(meta.path)
async def update_file_meta(self, path: str, **fields) -> FileMetadata | None:
"""Patch an existing meta (model_copy + upsert). No-op if path unknown."""
existing = self._nodes.get(path)
if existing is None:
return None
updated = existing.model_copy(update=fields)
await self.upsert_file_meta(updated)
return updated
async def delete_file_meta(self, path: str) -> FileMetadata | None:
"""Persist deletion + drop from index. Also drops the file's edges.
Returns prior meta if any.
"""
await self._persist_delete_meta(path)
await self._persist_delete_edges(path)
existing = self._nodes.pop(path, None)
if existing is not None:
stem = Path(existing.path).stem
self._stems.get(stem, set()).discard(existing.path)
if not self._stems.get(stem):
self._stems.pop(stem, None)
for edge in self._edges.pop(path, []):
tgt = _target_stem(edge.target)
self._backlinks.get(tgt, set()).discard(path)
if not self._backlinks.get(tgt):
self._backlinks.pop(tgt, None)
return existing
def get_file_meta(self, path: str) -> FileMetadata | None:
return self._nodes.get(path)
@property
def nodes(self) -> dict[str, FileMetadata]:
return self._nodes
def __len__(self) -> int:
return len(self._nodes)
def __contains__(self, path: str) -> bool:
return path in self._nodes
# -- Edge CRUD ----------------------------------------------------------
async def upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
"""Persist + reindex the edge list for `path`. Replaces any prior set."""
await self._persist_upsert_edges(path, edges)
# Drop old backlink contributions from this source.
for edge in self._edges.get(path, []):
tgt = _target_stem(edge.target)
self._backlinks.get(tgt, set()).discard(path)
if not self._backlinks.get(tgt):
self._backlinks.pop(tgt, None)
if edges:
self._edges[path] = list(edges)
for edge in edges:
self._backlinks[_target_stem(edge.target)].add(path)
else:
self._edges.pop(path, None)
def get_edges(self, path: str) -> list[FileEdge]:
"""All edges originating from `path` (raw — not resolved)."""
return list(self._edges.get(path, ()))
async def upsert_parsed(self, parsed: ParsedFile) -> None:
"""Single fan-out entry point for the watcher's parse pass.
Persists meta, edges, then chunks — the watcher gets one call and
the store handles the three sub-payloads in order.
"""
await self.upsert_file_meta(FileMetadata(
file=parsed.file,
path=parsed.path,
st_mtime=parsed.st_mtime,
metadata=parsed.metadata,
))
await self.upsert_edges(parsed.path, parsed.edges)
await self.upsert_chunks(parsed.path, parsed.chunks)
# -- Link queries (sync, in-memory) -------------------------------------
def get_links(self, path: str) -> list[tuple[FileMetadata, FileEdge]]:
"""Files this `path` links TO (resolved only).
Returns a list of `(target_meta, edge)` pairs — predicate / anchor /
source / confidence are preserved on the edge so callers can reason
about typed graph structure. The same target file can appear multiple
times if linked via several edges (e.g. plain `[[X]]` plus
`[author:: [[X]]]`); caller dedupes by `target_meta.path` if needed.
"""
if path not in self._nodes:
return []
out: list[tuple[FileMetadata, FileEdge]] = []
for edge in self._edges.get(path, ()):
hit = self.resolve_wikilink(edge.target)
if hit is not None and hit in self._nodes:
out.append((self._nodes[hit], edge))
return out
def get_backlinks(self, path: str) -> list[tuple[FileMetadata, FileEdge]]:
"""Files that link TO `path` (resolved only).
Returns a list of `(source_meta, edge)` pairs — `edge` is the
specific FileEdge on the source file that resolves to `path`. A
single source can appear multiple times if it points at `path` via
multiple edges.
"""
if path not in self._nodes:
return []
stem = Path(path).stem
candidates = self._backlinks.get(stem, set())
out: list[tuple[FileMetadata, FileEdge]] = []
for src in candidates:
src_meta = self._nodes.get(src)
if src_meta is None:
continue
for edge in self._edges.get(src, ()):
if self.resolve_wikilink(edge.target) == path:
out.append((src_meta, edge))
return out
def get_paths_by_stem(self, stem: str) -> list[str]:
return sorted(self._stems.get(stem, set()))
# -- Wikilink resolution ------------------------------------------------
def resolve_wikilink(self, wikilink: str) -> str | None:
target = wikilink.strip()
if not target:
return None
if "/" in target or target.endswith(".md"):
if self.vault_root is None:
return None
candidate = target if target.endswith(".md") else f"{target}.md"
abs_candidate = str((self.vault_root / candidate).resolve())
if abs_candidate in self._nodes:
return abs_candidate
return None
candidates = self.wikilink_candidates(target)
if len(candidates) == 1:
return candidates[0]
if len(candidates) > 1:
self.logger.warning(
f"Wikilink [[{target}]] is ambiguous, candidates: {candidates}",
)
return None
def resolve_wikilinks(self, wikilinks: list[str]) -> tuple[list[str], list[str]]:
"""Return (resolved_paths, dangling_targets)."""
resolved: list[str] = []
dangling: list[str] = []
for link in wikilinks:
hit = self.resolve_wikilink(link)
if hit is None:
dangling.append(link)
else:
resolved.append(hit)
return resolved, dangling
def wikilink_candidates(self, target: str) -> list[str]:
"""Paths `[[target]]` would resolve to (folder-note hit wins)."""
folder_hits = self._folder_notes(target)
if folder_hits:
return folder_hits
return self.get_paths_by_stem(target)
def collisions_after_create(self, proposed_path: str | Path) -> list[str]:
"""Existing paths that would conflict with adding `proposed_path`."""
p = Path(proposed_path)
stem = p.stem
proposed_abs = str(p.resolve())
is_folder_note = p.parent.name == stem
existing_folder_notes = [path for path in self._folder_notes(stem) if path != proposed_abs]
existing_stems = [path for path in self.get_paths_by_stem(stem) if path != proposed_abs]
if is_folder_note:
return existing_folder_notes
return existing_folder_notes + [sp for sp in existing_stems if sp not in existing_folder_notes]
def all_ambiguous_wikilinks(self) -> dict[str, list[str]]:
"""Stems whose `[[stem]]` form has >1 candidate."""
out: dict[str, list[str]] = {}
for stem in self._stems:
cands = self.wikilink_candidates(stem)
if len(cands) > 1:
out[stem] = cands
return out
def _folder_notes(self, stem: str) -> list[str]:
return sorted(p for p in self._stems.get(stem, ()) if Path(p).parent.name == stem)
# -- Graph-walk retrieval primitives ------------------------------------
def expand_neighbors(
self,
seed_paths: Iterable[str],
depth: int = 1,
direction: str = "both",
per_node_cap: int | None = 50,
) -> dict[str, int]:
"""BFS expansion over wikilink edges. Returns `{path: hop_distance}`."""
if direction not in ("out", "in", "both"):
raise ValueError(f"direction must be one of out/in/both, got {direction!r}")
if depth < 0:
raise ValueError(f"depth must be >= 0, got {depth}")
seen: dict[str, int] = {}
frontier: deque[tuple[str, int]] = deque()
for path in seed_paths:
if path in self._nodes and path not in seen:
seen[path] = 0
frontier.append((path, 0))
while frontier:
path, dist = frontier.popleft()
if dist >= depth:
continue
neighbors: list[str] = []
if direction in ("out", "both"):
neighbors.extend(m.path for m, _ in self.get_links(path))
if direction in ("in", "both"):
neighbors.extend(m.path for m, _ in self.get_backlinks(path))
if per_node_cap is not None and len(neighbors) > per_node_cap:
neighbors = neighbors[:per_node_cap]
for nb in neighbors:
if nb in seen:
continue
seen[nb] = dist + 1
frontier.append((nb, dist + 1))
return seen
def subgraph_score(
self,
seed_paths: Iterable[str],
decay: float = 0.5,
depth: int = 1,
direction: str = "both",
per_node_cap: int | None = 50,
) -> dict[str, float]:
"""Decayed score per path: seed=1.0, 1-hop=decay, 2-hop=decay²..."""
if not (0.0 <= decay <= 1.0):
raise ValueError(f"decay must be in [0, 1], got {decay}")
hops = self.expand_neighbors(seed_paths, depth, direction, per_node_cap)
return {path: decay ** hop for path, hop in hops.items()}
def extract_anchor_paths(self, text: str) -> list[str]:
"""Pull `[[X]]` anchors from `text` and resolve each (deduped)."""
seen: set[str] = set()
out: list[str] = []
for raw in extract_wikilinks(text):
hit = self.resolve_wikilink(raw)
if hit is not None and hit not in seen:
seen.add(hit)
out.append(hit)
return out
# -- Filter compilation -------------------------------------------------
def filter(
self,
paths: list[str] | None = None,
tags: list[str] | None = None,
exclude_paths: list[str] | None = None,
) -> ChunkFilter:
"""Compile user-facing intent into a path-set filter for chunk search."""
cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
if cf.is_empty():
return cf
cf.resolved_paths = {p for p, m in self._nodes.items() if cf.match_metadata(p, m.metadata)}
return cf
# -- Keyword scoring utility --------------------------------------------
@staticmethod
def _score_keyword_match(query: str, text: str) -> float:
words = query.split()
if not words:
return 0.0
query_lower = query.lower()
words_lower = [w.lower() for w in words]
text_lower = text.lower()
n_words = len(words)
match_count = sum(1 for w in words_lower if w in text_lower)
if match_count == 0:
return 0.0
base_score = match_count / n_words
phrase_bonus = 0.2 if n_words > 1 and query_lower in text_lower else 0.0
return min(1.0, base_score + phrase_bonus)
# -- Filter utility (chunk side) ----------------------------------------
@staticmethod
def _apply_filter(chunks: list[FileChunk], chunk_filter: ChunkFilter | None) -> list[FileChunk]:
if chunk_filter is None or chunk_filter.resolved_paths is None:
return chunks
return [c for c in chunks if chunk_filter.match_path(c.path)]
# -- Default file-meta persistence (sidecar JSONL) ----------------------
@property
def _file_metas_path(self) -> Path:
return self.db_path / f"{self.store_name}_files.jsonl"
async def _persist_upsert_meta(self, meta: FileMetadata) -> None:
"""Default: full rewrite of the sidecar jsonl. Backends with proper
relational storage (sqlite) override this for per-row UPSERT."""
# Snapshot current in-memory state with the new entry merged in.
# Note: in-memory not yet updated by caller — we add `meta` ourselves.
snapshot = {**self._nodes, meta.path: meta}
self._write_metas_jsonl(snapshot.values())
async def _persist_delete_meta(self, path: str) -> None:
"""Default: full rewrite of the sidecar jsonl, omitting `path`."""
snapshot = {p: m for p, m in self._nodes.items() if p != path}
self._write_metas_jsonl(snapshot.values())
def _iter_persisted_metas(self) -> Iterable[FileMetadata]:
"""Default: read all rows from the sidecar jsonl."""
path = self._file_metas_path
if not path.exists():
return []
out: list[FileMetadata] = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
out.append(FileMetadata.model_validate_json(line))
except Exception as e:
self.logger.warning(f"Bad row in {path}: {e}")
return out
def _write_metas_jsonl(self, metas: Iterable[FileMetadata]) -> None:
path = self._file_metas_path
lines = [m.model_dump_json() for m in metas]
content = "\n".join(lines)
tmp = path.with_suffix(".tmp")
try:
tmp.write_text(content, encoding="utf-8")
tmp.replace(path)
except Exception as e:
self.logger.error(f"Failed to write {path}: {e}")
raise
finally:
if tmp.exists():
tmp.unlink()
# -- Default edge persistence (sidecar JSONL) ---------------------------
@property
def _edges_path(self) -> Path:
return self.db_path / f"{self.store_name}_edges.jsonl"
async def _persist_upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
"""Default: full rewrite of the edge sidecar with `path`'s row replaced."""
snapshot = dict(self._edges)
if edges:
snapshot[path] = list(edges)
else:
snapshot.pop(path, None)
self._write_edges_jsonl(snapshot)
async def _persist_delete_edges(self, path: str) -> None:
"""Default: full rewrite of the edge sidecar omitting `path`."""
snapshot = {p: e for p, e in self._edges.items() if p != path}
self._write_edges_jsonl(snapshot)
def _iter_persisted_edges(self) -> Iterable[tuple[str, list[FileEdge]]]:
"""Default: yield (path, edges) rows from the sidecar jsonl."""
path = self._edges_path
if not path.exists():
return []
out: list[tuple[str, list[FileEdge]]] = []
import json as _json
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
row = _json.loads(line)
edges = [FileEdge.model_validate(e) for e in row.get("edges", [])]
out.append((row["path"], edges))
except Exception as e:
self.logger.warning(f"Bad edge row in {path}: {e}")
return out
def _write_edges_jsonl(self, edges_by_path: dict[str, list[FileEdge]]) -> None:
import json as _json
path = self._edges_path
lines = [
_json.dumps(
{"path": p, "edges": [e.model_dump(exclude_none=True) for e in edges]},
ensure_ascii=False,
)
for p, edges in edges_by_path.items()
]
content = "\n".join(lines)
tmp = path.with_suffix(".tmp")
try:
tmp.write_text(content, encoding="utf-8")
tmp.replace(path)
except Exception as e:
self.logger.error(f"Failed to write {path}: {e}")
raise
finally:
if tmp.exists():
tmp.unlink()
# -- Abstract chunk APIs ------------------------------------------------
@abstractmethod
async def clear_all(self):
"""Clear all indexed data (chunks + file metas)."""
@abstractmethod
async def upsert_chunks(self, path: str, chunks: list[FileChunk]):
"""Insert or update all chunks for a file path. Embeddings pre-attached."""
@abstractmethod
async def delete_chunks(self, path: str):
"""Delete all chunks for a file path."""
@abstractmethod
async def get_chunks(self, path: str) -> list[FileChunk]:
"""All chunks for a file path."""
@abstractmethod
async def get_chunks_by_paths(self, paths: Iterable[str]) -> list[FileChunk]:
"""Batch fetch chunks across many paths (used by graph-walk retrieval)."""
@abstractmethod
async def vector_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Vector similarity search."""
@abstractmethod
async def keyword_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Full-text/keyword search."""

98
reme2/component/chunk_store/local_chunk_store.py → reme2/component/file_store/local_file_store.py
View file

@ -1,19 +1,32 @@
"""Pure-Python chunk storage with JSONL persistence."""
"""Pure-Python in-memory store with on-close JSONL persistence.
Design (per project clarification):
- During runtime, ALL state lives in memory — meta + edges + chunks.
Per-write disk I/O is suppressed; `_persist_upsert_meta` / `_persist_upsert_edges`
are no-ops so each upsert_* costs only a dict mutation.
- On `_start`, load the JSONL sidecars into memory.
- On `_close`, flush the full in-memory snapshot back to JSONL.
Trade-off: lower write latency, but a hard crash drops anything since
the last clean shutdown. For larger / write-critical workloads, use
`SqliteFileStore` instead (per-write transaction).
"""
import json
from pathlib import Path
from typing import Iterable
import numpy as np
from .base_chunk_store import BaseChunkStore
from .base_file_store import BaseFileStore
from ..component_registry import R
from ...schema import ChunkFilter, FileChunk
from ...schema import ChunkFilter, FileChunk, FileEdge, FileMetadata
from ...utils import batch_cosine_similarity
@R.register("local")
class LocalChunkStore(BaseChunkStore):
"""In-memory chunk storage with JSONL disk persistence."""
class LocalFileStore(BaseFileStore):
"""In-memory chunk + meta + edge store with deferred JSONL persistence."""
def __init__(self, encoding: str = "utf-8", **kwargs):
super().__init__(**kwargs)
@ -24,7 +37,6 @@ class LocalChunkStore(BaseChunkStore):
# -- Persistence helpers ------------------------------------------------
async def _load_chunks(self) -> None:
"""Load chunks from JSONL file into memory."""
if not self._chunks_file.exists():
return
try:
@ -39,7 +51,6 @@ class LocalChunkStore(BaseChunkStore):
self.logger.warning(f"Failed to load chunks: {e}")
async def _save_chunks(self) -> None:
"""Persist chunks to JSONL file with atomic write."""
lines = [json.dumps(c.model_dump(mode="json"), ensure_ascii=False) for c in self._chunks.values()]
content = "\n".join(lines)
temp_path = self._chunks_file.with_suffix(".tmp")
@ -56,30 +67,52 @@ class LocalChunkStore(BaseChunkStore):
# -- Lifecycle ----------------------------------------------------------
async def _start(self) -> None:
"""Load persisted data into memory."""
await self._load_chunks()
self.logger.info(f"LocalChunkStore '{self.store_name}' ready: {len(self._chunks)} chunks")
# Base loads metas + edges from sidecar JSONL via the default
# _iter_persisted_metas / _iter_persisted_edges implementations.
await super()._start()
self.logger.info(
f"LocalFileStore '{self.store_name}' ready: "
f"{len(self._nodes)} files, {sum(len(v) for v in self._edges.values())} edges, "
f"{len(self._chunks)} chunks",
)
async def _close(self) -> None:
"""Flush state to disk and clear memory."""
await self._save_chunks()
# Flush the full in-memory snapshot before tearing down state.
try:
await self._save_chunks()
self._write_metas_jsonl(self._nodes.values())
self._write_edges_jsonl(self._edges)
except Exception as e:
self.logger.error(f"Failed to flush LocalFileStore '{self.store_name}': {e}")
self._chunks.clear()
await super()._close()
# -- Persistence overrides: no-op (we flush on close) -------------------
async def _persist_upsert_meta(self, meta: FileMetadata) -> None:
pass
async def _persist_delete_meta(self, path: str) -> None:
pass
async def _persist_upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
pass
async def _persist_delete_edges(self, path: str) -> None:
pass
# -- Write operations ---------------------------------------------------
async def upsert_chunks(self, path: str, chunks: list[FileChunk]) -> None:
"""Insert or update a file and its chunks."""
"""Insert or update a file's chunks. Chunks arrive embedded."""
await self.delete_chunks(path)
if not chunks:
return
chunks = await self.get_chunk_embeddings(chunks)
for chunk in chunks:
self._chunks[chunk.id] = chunk
async def delete_chunks(self, path: str) -> None:
"""Delete a file and all its chunks."""
to_delete = [cid for cid, chunk in self._chunks.items() if chunk.path == path]
for cid in to_delete:
del self._chunks[cid]
@ -87,20 +120,26 @@ class LocalChunkStore(BaseChunkStore):
# -- Read operations ----------------------------------------------------
async def get_chunks(self, path: str) -> list[FileChunk]:
"""Get all chunks for a file, sorted by start_line."""
chunks = [chunk for chunk in self._chunks.values() if chunk.path == path]
chunks.sort(key=lambda c: c.start_line)
return chunks
async def get_chunks_by_paths(self, paths: Iterable[str]) -> list[FileChunk]:
wanted = set(paths)
if not wanted:
return []
chunks = [c for c in self._chunks.values() if c.path in wanted]
chunks.sort(key=lambda c: (c.path, c.start_line))
return chunks
# -- Search operations --------------------------------------------------
async def vector_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Cosine-similarity vector search over in-memory embeddings."""
if not self.vector_enabled or not query:
return []
@ -147,12 +186,11 @@ class LocalChunkStore(BaseChunkStore):
return results[:limit]
async def keyword_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Keyword search via substring matching."""
if not self.fts_enabled or not query:
return []
@ -182,7 +220,13 @@ class LocalChunkStore(BaseChunkStore):
return results[:limit]
async def clear_all(self) -> None:
"""Clear all indexed data from memory and disk."""
"""Clear all in-memory state and on-disk JSONL sidecars."""
self._chunks.clear()
self._nodes.clear()
self._edges.clear()
self._stems.clear()
self._backlinks.clear()
await self._save_chunks()
self.logger.info(f"Cleared all data from LocalChunkStore '{self.store_name}'")
self._write_metas_jsonl([])
self._write_edges_jsonl({})
self.logger.info(f"Cleared all data from LocalFileStore '{self.store_name}'")

352
reme2/component/chunk_store/sqlite_chunk_store.py → reme2/component/file_store/sqlite_file_store.py
View file

@ -1,22 +1,24 @@
"""SQLite chunk storage backend."""
"""SQLite file store backend (chunks + file metadata)."""
import json
import sqlite3
import struct
import time
from typing import Iterable
from .base_chunk_store import BaseChunkStore
from .base_file_store import BaseFileStore
from ..component_registry import R
from ...schema import ChunkFilter, FileChunk
from ...schema import ChunkFilter, FileChunk, FileEdge, FileMetadata
@R.register("sqlite")
class SqliteChunkStore(BaseChunkStore):
"""SQLite chunk storage with vector and full-text search.
class SqliteFileStore(BaseFileStore):
"""SQLite file store with vector + full-text search.
Uses sqlite-vec for vector similarity search and FTS5 with trigram
tokenizer for keyword search. Falls back to LIKE-based substring
search for short query terms.
tokenizer for keyword search. File metadata lives in a relational
table (`files_{store_name}`) with native UPSERT — overrides the
base class's default JSONL sidecar.
"""
def __init__(self, vec_ext_path: str = "", **kwargs):
@ -38,13 +40,21 @@ class SqliteChunkStore(BaseChunkStore):
def fts_table(self) -> str:
return f"chunks_fts_{self.store_name}"
@property
def files_table(self) -> str:
return f"files_{self.store_name}"
@property
def edges_table(self) -> str:
return f"edges_{self.store_name}"
@staticmethod
def vector_to_blob(embedding: list[float]) -> bytes:
return struct.pack(f"{len(embedding)}f", *embedding)
# -- Lifecycle ----------------------------------------------------------
async def _start(self, app_context=None) -> None:
async def _start(self) -> None:
self.conn = sqlite3.connect(self.db_path / "reme.db", check_same_thread=False)
if self.vector_enabled:
@ -84,13 +94,14 @@ class SqliteChunkStore(BaseChunkStore):
self.conn.enable_load_extension(False)
await self._create_tables()
self.logger.info(f"SqliteChunkStore '{self.store_name}' ready: db={self.db_path / 'reme.db'}")
await super()._start(app_context)
self.logger.info(f"SqliteFileStore '{self.store_name}' ready: db={self.db_path / 'reme.db'}")
await super()._start()
async def _create_tables(self) -> None:
cursor = self.conn.cursor()
try:
cursor.execute(f"""
cursor.execute(
f"""
CREATE TABLE IF NOT EXISTS {self.chunks_table} (
id TEXT PRIMARY KEY,
path TEXT,
@ -101,23 +112,26 @@ class SqliteChunkStore(BaseChunkStore):
embedding TEXT,
updated_at INTEGER
)
""")
""",
)
cursor.execute(
f"CREATE INDEX IF NOT EXISTS idx_{self.chunks_table}_path "
f"ON {self.chunks_table}(path)",
f"CREATE INDEX IF NOT EXISTS idx_{self.chunks_table}_path " f"ON {self.chunks_table}(path)",
)
if self.vector_enabled:
cursor.execute(f"""
cursor.execute(
f"""
CREATE VIRTUAL TABLE IF NOT EXISTS {self.vector_table} USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[{self.embedding_dim}]
)
""")
""",
)
self.logger.info(f"Created vector table (dims={self.embedding_dim})")
if self.fts_enabled:
cursor.execute(f"""
cursor.execute(
f"""
CREATE VIRTUAL TABLE IF NOT EXISTS {self.fts_table} USING fts5(
text,
id UNINDEXED,
@ -126,9 +140,30 @@ class SqliteChunkStore(BaseChunkStore):
end_line UNINDEXED,
tokenize='trigram'
)
""")
""",
)
self.logger.info("Created FTS5 table with trigram tokenizer")
cursor.execute(
f"""
CREATE TABLE IF NOT EXISTS {self.files_table} (
path TEXT PRIMARY KEY,
file TEXT,
st_mtime REAL,
metadata TEXT
)
""",
)
cursor.execute(
f"""
CREATE TABLE IF NOT EXISTS {self.edges_table} (
path TEXT PRIMARY KEY,
edges TEXT
)
""",
)
self.conn.commit()
except Exception as e:
self.logger.error(f"Failed to create tables: {e}")
@ -150,7 +185,8 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute("BEGIN")
old_ids = [
row[0] for row in cursor.execute(
row[0]
for row in cursor.execute(
f"SELECT id FROM {self.chunks_table} WHERE path = ?",
(path,),
).fetchall()
@ -165,7 +201,6 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute(f"DELETE FROM {self.fts_table} WHERE path = ?", (path,))
if chunks:
chunks = await self.get_chunk_embeddings(chunks)
now = int(time.time() * 1000)
for chunk in chunks:
cursor.execute(
@ -212,8 +247,10 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute("BEGIN")
chunk_ids = [
row[0] for row in cursor.execute(
f"SELECT id FROM {self.chunks_table} WHERE path = ?", (path,),
row[0]
for row in cursor.execute(
f"SELECT id FROM {self.chunks_table} WHERE path = ?",
(path,),
).fetchall()
]
@ -254,15 +291,17 @@ class SqliteChunkStore(BaseChunkStore):
embedding = json.loads(emb_str)
except (json.JSONDecodeError, TypeError):
pass
chunks.append(FileChunk(
id=chunk_id,
path=path_val,
start_line=start,
end_line=end,
text=text,
hash=hash_val,
embedding=embedding,
))
chunks.append(
FileChunk(
id=chunk_id,
path=path_val,
start_line=start,
end_line=end,
text=text,
hash=hash_val,
embedding=embedding,
),
)
return chunks
except Exception as e:
self.logger.error(f"Failed to get chunks for {path}: {e}")
@ -270,13 +309,62 @@ class SqliteChunkStore(BaseChunkStore):
finally:
cursor.close()
async def get_chunks_by_paths(self, paths: Iterable[str]) -> list[FileChunk]:
"""Batch fetch chunks across many paths.
Splits into ≤900-placeholder batches to stay under sqlite's
SQLITE_MAX_VARIABLE_NUMBER (default 999, leave headroom).
"""
wanted = list({p for p in paths})
if not wanted:
return []
cursor = self.conn.cursor()
try:
chunks: list[FileChunk] = []
BATCH = 900
for offset in range(0, len(wanted), BATCH):
batch = wanted[offset : offset + BATCH]
placeholders = ",".join("?" * len(batch))
cursor.execute(
f"""SELECT id, path, start_line, end_line, text, hash, embedding
FROM {self.chunks_table} WHERE path IN ({placeholders})
ORDER BY path, start_line""",
batch,
)
for row in cursor.fetchall():
chunk_id, path_val, start_line, end, text, hash_val, emb_str = row
embedding = None
if emb_str:
try:
embedding = json.loads(emb_str)
except (json.JSONDecodeError, TypeError):
pass
chunks.append(
FileChunk(
id=chunk_id,
path=path_val,
start_line=start_line,
end_line=end,
text=text,
hash=hash_val,
embedding=embedding,
),
)
return chunks
except Exception as e:
self.logger.error(f"Failed to batch get chunks ({len(wanted)} paths): {e}")
return []
finally:
cursor.close()
# -- Search helpers -----------------------------------------------------
@staticmethod
def _sanitize_fts_query(query: str) -> str:
if not query:
return ""
special_chars = list('*?:^()[]{}\'"`|+-=<>!@#$%&\\/,;')
special_chars = list("*?:^()[]{}'\"`|+-=<>!@#$%&\\/,;")
cleaned = query
for ch in special_chars:
cleaned = cleaned.replace(ch, " ")
@ -296,10 +384,10 @@ class SqliteChunkStore(BaseChunkStore):
# -- Search operations --------------------------------------------------
async def vector_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
if not self.vector_enabled or not query:
return []
@ -325,15 +413,17 @@ class SqliteChunkStore(BaseChunkStore):
chunks = []
for cid, path, start, end, text, dist in cursor.fetchall():
score = max(0.0, 1.0 - dist / 2.0)
chunks.append(FileChunk(
id=cid,
path=path,
start_line=start,
end_line=end,
text=text,
hash="",
scores={"vector": score, "score": score},
))
chunks.append(
FileChunk(
id=cid,
path=path,
start_line=start,
end_line=end,
text=text,
hash="",
scores={"vector": score, "score": score},
),
)
chunks = self._apply_filter(chunks, chunk_filter)
chunks.sort(key=lambda c: c.score, reverse=True)
@ -345,10 +435,10 @@ class SqliteChunkStore(BaseChunkStore):
cursor.close()
async def keyword_search(
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
self,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
if not self.fts_enabled or not query:
return []
@ -391,15 +481,17 @@ class SqliteChunkStore(BaseChunkStore):
chunks = []
for cid, path, start, end, text, rank in cursor.fetchall():
score = max(0.0, 1.0 / (1.0 + abs(rank)))
chunks.append(FileChunk(
id=cid,
path=path,
start_line=start,
end_line=end,
text=text,
hash="",
scores={"keyword": score, "score": score},
))
chunks.append(
FileChunk(
id=cid,
path=path,
start_line=start,
end_line=end,
text=text,
hash="",
scores={"keyword": score, "score": score},
),
)
chunks.sort(key=lambda c: c.score, reverse=True)
return chunks
except Exception as e:
@ -437,15 +529,17 @@ class SqliteChunkStore(BaseChunkStore):
if score == 0.0:
continue
chunks.append(FileChunk(
id=cid,
path=path,
start_line=start,
end_line=end,
text=text,
hash="",
scores={"keyword": score, "score": score},
))
chunks.append(
FileChunk(
id=cid,
path=path,
start_line=start,
end_line=end,
text=text,
hash="",
scores={"keyword": score, "score": score},
),
)
chunks.sort(key=lambda c: c.score, reverse=True)
return chunks[:limit]
@ -466,6 +560,8 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute(f"DELETE FROM {self.vector_table}")
if self.fts_enabled:
cursor.execute(f"DELETE FROM {self.fts_table}")
cursor.execute(f"DELETE FROM {self.files_table}")
cursor.execute(f"DELETE FROM {self.edges_table}")
cursor.execute("COMMIT")
except Exception as e:
cursor.execute("ROLLBACK")
@ -473,4 +569,124 @@ class SqliteChunkStore(BaseChunkStore):
raise
finally:
cursor.close()
self.logger.info(f"Cleared all data from SqliteChunkStore '{self.store_name}'")
self.logger.info(f"Cleared all data from SqliteFileStore '{self.store_name}'")
# -- File-meta persistence (overrides default JSONL sidecar) ------------
async def _persist_upsert_meta(self, meta: FileMetadata) -> None:
cursor = self.conn.cursor()
try:
cursor.execute(
f"""INSERT INTO {self.files_table}
(path, file, st_mtime, metadata)
VALUES (?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
file = excluded.file,
st_mtime = excluded.st_mtime,
metadata = excluded.metadata""",
(
meta.path,
meta.file,
meta.st_mtime,
json.dumps(meta.metadata, ensure_ascii=False, default=str),
),
)
self.conn.commit()
except Exception as e:
self.conn.rollback()
self.logger.error(f"Failed to upsert file meta {meta.path}: {e}")
raise
finally:
cursor.close()
async def _persist_delete_meta(self, path: str) -> None:
cursor = self.conn.cursor()
try:
cursor.execute(f"DELETE FROM {self.files_table} WHERE path = ?", (path,))
self.conn.commit()
except Exception as e:
self.conn.rollback()
self.logger.error(f"Failed to delete file meta {path}: {e}")
raise
finally:
cursor.close()
def _iter_persisted_metas(self) -> Iterable[FileMetadata]:
if self.conn is None:
return []
cursor = self.conn.cursor()
try:
cursor.execute(
f"SELECT path, file, st_mtime, metadata "
f"FROM {self.files_table}",
)
out: list[FileMetadata] = []
for path, file, st_mtime, metadata_json in cursor.fetchall():
try:
out.append(
FileMetadata(
path=path,
file=file or "",
st_mtime=st_mtime or 0.0,
metadata=json.loads(metadata_json) if metadata_json else {},
),
)
except Exception as e:
self.logger.warning(f"Bad file meta row for {path}: {e}")
return out
finally:
cursor.close()
# -- Edge persistence (overrides default JSONL sidecar) -----------------
async def _persist_upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
cursor = self.conn.cursor()
try:
if edges:
cursor.execute(
f"""INSERT INTO {self.edges_table} (path, edges)
VALUES (?, ?)
ON CONFLICT(path) DO UPDATE SET edges = excluded.edges""",
(path, json.dumps(
[e.model_dump(exclude_none=True) for e in edges],
ensure_ascii=False,
)),
)
else:
cursor.execute(f"DELETE FROM {self.edges_table} WHERE path = ?", (path,))
self.conn.commit()
except Exception as e:
self.conn.rollback()
self.logger.error(f"Failed to upsert edges for {path}: {e}")
raise
finally:
cursor.close()
async def _persist_delete_edges(self, path: str) -> None:
cursor = self.conn.cursor()
try:
cursor.execute(f"DELETE FROM {self.edges_table} WHERE path = ?", (path,))
self.conn.commit()
except Exception as e:
self.conn.rollback()
self.logger.error(f"Failed to delete edges for {path}: {e}")
raise
finally:
cursor.close()
def _iter_persisted_edges(self) -> Iterable[tuple[str, list[FileEdge]]]:
if self.conn is None:
return []
cursor = self.conn.cursor()
try:
cursor.execute(f"SELECT path, edges FROM {self.edges_table}")
out: list[tuple[str, list[FileEdge]]] = []
for path, edges_json in cursor.fetchall():
try:
raws = json.loads(edges_json) if edges_json else []
out.append((path, [FileEdge.model_validate(e) for e in raws]))
except Exception as e:
self.logger.warning(f"Bad edge row for {path}: {e}")
return out
finally:
cursor.close()

349
reme2/component/file_watcher/base_file_watcher.py
View file

@ -1,82 +1,95 @@
"""Base file watcher with watchfiles integration."""
"""Base file watcher with watchfiles integration.
Per change → single-step pipeline:
added/modified → existing = file_store.get_chunks(path)
parsed = parser.parse(path, existing_chunks=existing)
file_store.upsert_parsed(parsed)
deleted → file_store.delete_chunks + delete_file_meta
The parser owns: chunking, edge extraction, and embedding (with hash-diff
cache via the `existing_chunks` argument). The file_store owns: persistence
of meta + edges + chunks (atomic-from-the-caller's-POV via `upsert_parsed`).
The watcher owns: scheduling, cancel-on-modify, retry/timeout, and
startup recovery (re-parsing files whose mtime drifted while offline).
"""
import asyncio
import time
from pathlib import Path
from watchfiles import Change, awatch
from ..base_component import BaseComponent
from ..chunk_store import BaseChunkStore
from ..file_parser import BaseFileParser
from ..file_store import BaseFileStore
from ...enumeration import ComponentEnum
from ...schema.file_graph import FileGraph
class BaseFileWatcher(BaseComponent):
"""Abstract base class for file watchers.
Provides file monitoring with:
- watchfiles integration for efficient change detection
- Parser-based file filtering
- Auto-restart on failure
- Optional index rebuild on start
"""
"""Watches a directory and feeds the file_store via single-step parse+upsert."""
component_type = ComponentEnum.FILE_WATCHER
_META_DIR = ".reme"
def __init__(
self,
watch_path: str,
recursive: bool = False,
debounce: int = 2000,
chunk_tokens: int = 400,
chunk_overlap: int = 80,
chunk_store: str = "default",
default_parser: str | None = None,
rebuild_index_on_start: bool = False,
poll_delay_ms: int = 2000,
**kwargs,
self,
watch_path: str,
recursive: bool = False,
debounce: int = 2000,
file_store: str = "default",
default_parser: str | None = None,
rebuild_index_on_start: bool = False,
poll_delay_ms: int = 2000,
parse_max_attempts: int = 3,
parse_retry_backoff: float = 2.0,
parse_task_timeout: float = 300.0,
**kwargs,
):
super().__init__(**kwargs)
self._chunk_store_name: str = chunk_store
self._file_store_name: str = file_store
self._default_parser_name: str | None = default_parser
self.chunk_store: BaseChunkStore | None = None
self.file_store: BaseFileStore | None = None
self._suffix_to_parser: dict[str, BaseFileParser] = {}
self._default_parser: BaseFileParser | None = None
self.watch_path: str = watch_path
self.recursive: bool = recursive
self.debounce: int = debounce
self.chunk_tokens: int = chunk_tokens
self.chunk_overlap: int = chunk_overlap
self.rebuild_index_on_start: bool = rebuild_index_on_start
self.poll_delay_ms: int = poll_delay_ms
self.file_graph: FileGraph = FileGraph()
self._stop_event = asyncio.Event()
self._watch_task: asyncio.Task | None = None
_META_DIR = ".reme"
# Parse pipeline configuration.
self._parse_max_attempts: int = max(1, int(parse_max_attempts))
self._parse_retry_backoff: float = float(parse_retry_backoff)
self._parse_task_timeout: float = float(parse_task_timeout)
self._tasks: dict[str, asyncio.Task] = {}
self._mgmt_lock: asyncio.Lock | None = None
self._run_lock: asyncio.Lock | None = None
self._last_failure: dict[str, float] = {}
@property
def _meta_path(self) -> Path:
return Path(self.watch_path) / self._META_DIR
return (Path(self.watch_path) / self._META_DIR).resolve()
@property
def _graph_path(self) -> Path:
return self._meta_path / "file_graph.json"
# -- Lifecycle ----------------------------------------------------------
async def _start(self):
"""Resolve chunk_store, load or build file_graph, and start watching."""
if self._chunk_store_name:
"""Resolve file_store + parsers, sync any disk drift, start watch loop."""
if self._file_store_name:
assert self.app_context is not None, "app_context must be provided"
stores = self.app_context.components.get(ComponentEnum.CHUNK_STORE, {})
if self._chunk_store_name not in stores:
raise ValueError(f"Chunk store '{self._chunk_store_name}' not found.")
store = stores[self._chunk_store_name]
if not isinstance(store, BaseChunkStore):
raise TypeError(f"Expected BaseChunkStore, got {type(store).__name__}")
self.chunk_store = store
stores = self.app_context.components.get(ComponentEnum.FILE_STORE, {})
if self._file_store_name not in stores:
raise ValueError(f"File store '{self._file_store_name}' not found.")
store = stores[self._file_store_name]
if not isinstance(store, BaseFileStore):
raise TypeError(f"Expected BaseFileStore, got {type(store).__name__}")
self.file_store = store
# Vault root drives explicit-path wikilink resolution.
self.file_store.set_vault_root(self.watch_path)
parsers = self.app_context.components.get(ComponentEnum.FILE_PARSER, {})
for parser in parsers.values():
@ -93,28 +106,16 @@ class BaseFileWatcher(BaseComponent):
self.logger.warning("No file parsers registered")
async def _initialize_and_watch():
await self._load_or_build_graph()
await self._initial_sync_and_recovery()
await self._watch_loop()
self._mgmt_lock = asyncio.Lock()
self._run_lock = asyncio.Lock()
self._last_failure.clear()
self._stop_event.clear()
self._watch_task = asyncio.create_task(_initialize_and_watch())
self.logger.info(f"Started watching: {self.watch_path}")
async def _load_or_build_graph(self) -> None:
graph_path = self._graph_path
if not self.rebuild_index_on_start and graph_path.exists():
self.file_graph = FileGraph.load(graph_path)
self.logger.info(f"Loaded file graph from {graph_path} ({len(self.file_graph)} nodes)")
else:
if self.rebuild_index_on_start:
self.logger.info("Rebuild on start enabled, scanning to build")
else:
self.logger.info("No persisted file graph found, scanning to build")
self.file_graph = FileGraph()
await self._scan_existing_files()
self.file_graph.save(graph_path)
self.logger.info(f"Saved file graph to {graph_path} ({len(self.file_graph)} nodes)")
async def _close(self):
"""Stop watching and release resources."""
self._stop_event.set()
@ -125,39 +126,94 @@ class BaseFileWatcher(BaseComponent):
except asyncio.CancelledError:
pass
# Cancel all in-flight parse tasks before tearing down state.
tasks = list(self._tasks.values())
for t in tasks:
if not t.done():
t.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
self._tasks.clear()
self._last_failure.clear()
self._watch_task = None
self._stop_event.clear()
self.chunk_store = None
self._mgmt_lock = None
self._run_lock = None
self.file_store = None
self._suffix_to_parser.clear()
self._default_parser = None
self.logger.info("Stopped watching")
async def _scan_existing_files(self) -> None:
if not self.chunk_store:
return
# -- Startup sync -------------------------------------------------------
async def _initial_sync_and_recovery(self) -> None:
"""At startup: file_store has its persisted state loaded already; we
diff against current disk for any changes that happened while we
were offline, then re-enqueue them through the parse pipeline."""
await self._sync_with_disk()
async def _sync_with_disk(self) -> bool:
"""Diff cached graph (in file_store) vs disk state; reindex changes."""
if self.file_store is None:
return False
watch_path = Path(self.watch_path)
if not watch_path.exists():
self.logger.warning(f"Watch path does not exist: {watch_path}")
return
return False
existing_files: set[tuple[Change, str]] = set()
on_disk: dict[str, float] = {}
if watch_path.is_file():
existing_files.add((Change.added, str(watch_path)))
on_disk[str(watch_path.resolve())] = watch_path.stat().st_mtime
elif watch_path.is_dir():
iterator = watch_path.rglob("*") if self.recursive else watch_path.iterdir()
for file_path in iterator:
if file_path.is_file() and self._watch_filter(str(file_path)):
existing_files.add((Change.added, str(file_path)))
for fp in iterator:
if not fp.is_file():
continue
abs_path = str(fp.resolve())
if not self._watch_filter(abs_path):
continue
if not self._get_parser(abs_path):
continue
try:
on_disk[abs_path] = fp.stat().st_mtime
except OSError:
continue
if existing_files:
self.logger.info(f"Scanning {len(existing_files)} existing files")
await self.on_changes(existing_files)
else:
self.logger.info("No existing files found")
cached_paths = set(self.file_store.nodes.keys())
disk_paths = set(on_disk.keys())
added = disk_paths - cached_paths
deleted = cached_paths - disk_paths
modified: set[str] = set()
for p in disk_paths & cached_paths:
cached_meta = self.file_store.get_file_meta(p)
if cached_meta is None or cached_meta.st_mtime != on_disk[p]:
modified.add(p)
unchanged = len(disk_paths & cached_paths) - len(modified)
if not (added or deleted or modified):
self.logger.info(f"Index up-to-date ({unchanged} files cached, 0 changes)")
return False
self.logger.info(
f"Incremental sync: +{len(added)} ~{len(modified)} -{len(deleted)} (unchanged {unchanged})",
)
changes: set[tuple[Change, str]] = set()
for p in deleted:
changes.add((Change.deleted, p))
for p in added:
changes.add((Change.added, p))
for p in modified:
changes.add((Change.modified, p))
await self.on_changes(changes)
return True
# -- Watch loop ---------------------------------------------------------
async def _interruptible_sleep(self, seconds: float):
"""Sleep that can be interrupted by stop_event."""
try:
await asyncio.wait_for(self._stop_event.wait(), timeout=seconds)
except asyncio.TimeoutError:
@ -173,12 +229,12 @@ class BaseFileWatcher(BaseComponent):
try:
self.logger.info(f"Starting watch on: {self.watch_path}")
async for changes in awatch(
self.watch_path,
watch_filter=lambda _, p: self._watch_filter(p),
recursive=self.recursive,
debounce=self.debounce,
poll_delay_ms=self.poll_delay_ms,
stop_event=self._stop_event,
self.watch_path,
watch_filter=lambda _, p: self._watch_filter(p),
recursive=self.recursive,
debounce=self.debounce,
poll_delay_ms=self.poll_delay_ms,
stop_event=self._stop_event,
):
if self._stop_event.is_set():
break
@ -195,22 +251,25 @@ class BaseFileWatcher(BaseComponent):
await self._interruptible_sleep(10)
def _watch_filter(self, path: str) -> bool:
return self._meta_path not in Path(path).parents
resolved = Path(path).resolve()
if resolved == self._meta_path:
return False
return self._meta_path not in resolved.parents
def _get_parser(self, path: str) -> BaseFileParser | None:
suffix = Path(path).suffix.lower()
return self._suffix_to_parser.get(suffix, self._default_parser)
# -- Change dispatch ----------------------------------------------------
async def on_changes(self, changes: set[tuple[Change, str]]) -> None:
if not self.chunk_store:
if self.file_store is None:
self.logger.warning("File store not initialized, skipping changes")
return
for change_type, path in changes:
try:
if change_type == Change.added:
await self._on_added(path)
elif change_type == Change.modified:
if change_type in (Change.added, Change.modified):
await self._on_modified(path)
elif change_type == Change.deleted:
await self._on_deleted(path)
@ -219,29 +278,115 @@ class BaseFileWatcher(BaseComponent):
except PermissionError:
self.logger.warning(f"Permission denied: {path}, skipping")
except Exception as e:
self.logger.error(f"Error processing {path}: {e}", exc_info=True)
async def _on_added(self, path: str) -> None:
parser = self._get_parser(path)
if not parser:
self.logger.debug(f"No parser for {path}, skipping")
return
file_meta, chunks = await parser.parse(path)
await self.chunk_store.upsert_chunks(path, chunks)
self.file_graph.create(file_meta)
self.logger.info(f"Added {path} ({len(chunks)} chunks)")
self.logger.opt(exception=True).error("Error processing {p}: {err}", p=path, err=str(e))
async def _on_modified(self, path: str) -> None:
if not Path(path).is_file():
return
parser = self._get_parser(path)
if not parser:
if parser is None:
self.logger.debug(f"No parser for {path}, skipping")
return
file_meta, chunks = await parser.parse(path)
await self.chunk_store.upsert_chunks(path, chunks)
self.file_graph.create(file_meta)
self.logger.info(f"Modified {path} ({len(chunks)} chunks)")
await self._submit_parse_task(path, parser)
async def _on_deleted(self, path: str) -> None:
await self.chunk_store.delete_chunks(path)
self.file_graph.delete(path)
# Cancel any in-flight parse task FIRST so a delayed cancellation
# can't race the cleanup writes below.
assert self.file_store is not None, "_on_deleted requires file_store"
await self._cancel_parse_task(path)
await self.file_store.delete_chunks(path)
await self.file_store.delete_file_meta(path)
self.logger.info(f"Deleted {path}")
# -- Parse task pipeline ------------------------------------------------
async def _submit_parse_task(self, path: str, parser: BaseFileParser) -> None:
"""Cancel any prior task + spawn a new one. Atomic under _mgmt_lock."""
assert self._mgmt_lock is not None, "_submit_parse_task before _start()"
async with self._mgmt_lock:
await self._cancel_locked(path)
self._tasks[path] = asyncio.create_task(
self._run_parse_task(path, parser),
)
async def _cancel_parse_task(self, path: str) -> None:
"""Cancel + await any in-flight parse task for `path`."""
assert self._mgmt_lock is not None, "_cancel_parse_task before _start()"
async with self._mgmt_lock:
await self._cancel_locked(path)
async def _cancel_locked(self, path: str) -> None:
"""Cancel + await the task for `path`. Caller holds `_mgmt_lock`."""
task = self._tasks.pop(path, None)
if task is not None and not task.done():
task.cancel()
try:
await task
except (asyncio.CancelledError, Exception):
pass
async def flush_parse_tasks(self) -> None:
"""Wait until all parse tasks finish (loops because retries spawn new ones)."""
while self._tasks:
await asyncio.gather(*list(self._tasks.values()), return_exceptions=True)
def pending_parse_count(self) -> int:
return sum(1 for t in self._tasks.values() if not t.done())
def failed_parse_paths(self) -> dict[str, float]:
return dict(self._last_failure)
async def _run_parse_task(self, path: str, parser: BaseFileParser) -> None:
"""One per-path parse task: get prior chunks → parse → upsert_parsed.
Hash diff happens inside the parser via `existing_chunks`: chunks
whose hash matches a stored chunk reuse the prior embedding; only
new-hash blocks hit the embedding API. The whole upsert is one
call to the file_store.
"""
try:
for attempt in range(1, self._parse_max_attempts + 1):
try:
assert self.file_store is not None
existing = await self.file_store.get_chunks(path)
parsed = await asyncio.wait_for(
parser.parse(path, existing_chunks=existing),
timeout=self._parse_task_timeout,
)
assert self._run_lock is not None
async with self._run_lock:
await asyncio.wait_for(
self.file_store.upsert_parsed(parsed),
timeout=self._parse_task_timeout,
)
self._last_failure.pop(path, None)
self.logger.info(
f"indexed {path}: chunks={len(parsed.chunks)} edges={len(parsed.edges)}",
)
return
except asyncio.CancelledError:
raise
except FileNotFoundError:
self.logger.debug(f"parse task {path}: file gone, skipping")
self._last_failure.pop(path, None)
return
except Exception as e:
self._last_failure[path] = time.time()
if attempt < self._parse_max_attempts:
backoff = self._parse_retry_backoff * attempt
self.logger.warning(
f"parse task {path} attempt {attempt} failed "
f"({type(e).__name__}: {e}); retry in {backoff:.1f}s",
)
await asyncio.sleep(backoff)
continue
self.logger.error(
f"parse task {path} giving up after {attempt} attempts: "
f"{type(e).__name__}: {e}",
)
return
finally:
if self._tasks.get(path) is asyncio.current_task():
self._tasks.pop(path, None)

17
reme2/component/job/base_job.py
View file

@ -1,4 +1,5 @@
"""Base job component for sequential step execution."""
from typing import TYPE_CHECKING
from ..base_component import BaseComponent
@ -18,11 +19,11 @@ class BaseJob(BaseComponent):
component_type = ComponentEnum.JOB
def __init__(
self,
description: str = "",
parameters: dict | None = None,
steps: list[ComponentConfig] | None = None,
**kwargs,
self,
description: str = "",
parameters: dict | None = None,
steps: list[ComponentConfig] | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.description = description
@ -32,14 +33,14 @@ class BaseJob(BaseComponent):
async def _start(self) -> None:
assert self.app_context is not None, "app_context must be provided"
for config in self.step_configs:
for raw in self.step_configs:
config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
if not config.backend:
raise ValueError(f"Step is missing the required 'backend' field")
step_cls = R.get(ComponentEnum.STEP, config.backend)
if not step_cls:
raise ValueError(
f"Step references an unregistered backend '{config.backend}' "
f"of type '{ComponentEnum.STEP}'",
f"Step references an unregistered backend '{config.backend}' " f"of type '{ComponentEnum.STEP}'",
)
params = config.model_dump()
params["app_context"] = self.app_context

6
reme2/component/prompt_handler.py
View file

@ -21,9 +21,9 @@ class PromptHandler:
self.language: str = language.strip()
def load_prompt_by_file(
self,
prompt_file_path: str | Path | None = None,
overwrite: bool = True,
self,
prompt_file_path: str | Path | None = None,
overwrite: bool = True,
) -> "PromptHandler":
"""Load prompts from a YAML or JSON file."""
if prompt_file_path is None:

30
reme2/component/runtime_response.py Normal file
View file

@ -0,0 +1,30 @@
"""Helpers for serializing Step output onto `RuntimeContext.response`.
Lives next to `runtime_context.py` because both are about the BaseStep
interface — the response side, specifically. Used by every Step that
returns a JSON-shaped payload (memory_*, sync, topic_create, the three
memory services).
Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP
dependency into `reme2/memory/` services that legitimately need to
serialize their results — the moved location breaks that cycle.
"""
from __future__ import annotations
import json
from datetime import date, datetime
def _to_jsonable(value):
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, dict):
return {k: _to_jsonable(v) for k, v in value.items()}
if isinstance(value, (list, tuple, set)):
return [_to_jsonable(v) for v in value]
return value
def _set_answer(context, payload) -> None:
context.response.answer = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2)

9
reme2/component/service/base_service.py
View file

@ -21,16 +21,13 @@ class BaseService(BaseComponent):
self.service = None
@abstractmethod
def build_service(self, app: "Application") -> None:
...
def build_service(self, app: "Application") -> None: ...
@abstractmethod
def add_job(self, job: BaseJob) -> None:
...
def add_job(self, job: BaseJob) -> None: ...
@abstractmethod
def start_service(self, app: "Application") -> None:
...
def start_service(self, app: "Application") -> None: ...
def add_jobs(self, app: "Application") -> None:
for name, job in app.context.jobs.items():

8
reme2/component/service/http_service.py
View file

@ -51,10 +51,10 @@ class HttpService(BaseService):
async def generate_stream() -> AsyncGenerator[bytes, None]:
async for chunk in execute_stream_task(
stream_queue=stream_queue,
task=task,
task_name=job.name,
output_format="bytes",
stream_queue=stream_queue,
task=task,
task_name=job.name,
output_format="bytes",
):
assert isinstance(chunk, bytes)
yield chunk

109
reme2/component/service/mcp_service.py
View file

@ -1,9 +1,20 @@
"""Model Context Protocol (MCP) service implementation."""
"""Model Context Protocol (MCP) service implementation.
Optionally exposes an HTTP sidecar (off by default) on a local-only
loopback port. The sidecar shares the running Application instance,
so callers (lifecycle hooks, scripts) see the live file_store without
booting a duplicate process.
"""
import asyncio
import json
import os
from contextlib import asynccontextmanager
from pathlib import Path
from typing import TYPE_CHECKING
import uvicorn
from fastapi import FastAPI, Request as FastAPIRequest
from fastmcp import FastMCP
from fastmcp.server.server import Transport
from fastmcp.tools import FunctionTool
@ -18,20 +29,54 @@ if TYPE_CHECKING:
from ..job import BaseJob
def _build_sidecar_app(app: "Application") -> FastAPI:
"""A minimal FastAPI app that exposes each registered job as POST /<job>."""
api = FastAPI(title=f"{app.config.app_name}-sidecar")
@api.get("/health")
async def health():
return {"status": "ok", "jobs": list(app.context.jobs.keys())}
for job_name in list(app.context.jobs.keys()):
# bind job_name via default arg to avoid late-binding in the closure
async def endpoint(request: FastAPIRequest, _name: str = job_name):
try:
payload = await request.json()
except Exception:
payload = {}
response = await app.run_job(_name, **(payload or {}))
return {"answer": response.answer, "success": response.success}
api.post(f"/{job_name}")(endpoint)
return api
@R.register("mcp")
class MCPService(BaseService):
"""Expose jobs as Model Context Protocol (MCP) tools."""
def __init__(
self,
transport: Transport = "sse",
host: str = REME_DEFAULT_HOST,
port: int = REME_DEFAULT_PORT,
**kwargs):
self,
transport: Transport = "sse",
host: str = REME_DEFAULT_HOST,
port: int = REME_DEFAULT_PORT,
sidecar_http: bool = False,
sidecar_http_host: str = "127.0.0.1",
sidecar_http_port: int = 8765,
sidecar_info_path: str = "",
**kwargs,
):
super().__init__(**kwargs)
self.transport: Transport = transport
self.host: str = host
self.port: int = port
self.sidecar_http: bool = sidecar_http
self.sidecar_http_host: str = sidecar_http_host
self.sidecar_http_port: int = sidecar_http_port
# Optional file path where {host, port} are dropped at startup so that
# external scripts (lifecycle hooks) can discover the sidecar URL.
self.sidecar_info_path: str = sidecar_info_path
def build_service(self, app: "Application") -> None:
@ -41,11 +86,59 @@ class MCPService(BaseService):
service_info = json.dumps({"host": self.host, "port": self.port})
os.environ[REME_SERVICE_INFO] = service_info
self.logger.info(f"ReMe MCP Service started: {REME_SERVICE_INFO}={service_info}")
yield
await app.close()
sidecar_task: asyncio.Task | None = None
if self.sidecar_http:
sidecar_task = asyncio.create_task(self._serve_sidecar(app))
try:
yield
finally:
if sidecar_task is not None:
sidecar_task.cancel()
try:
await sidecar_task
except (asyncio.CancelledError, Exception):
pass
self._cleanup_sidecar_info()
await app.close()
self.service = FastMCP(name=app.config.app_name, lifespan=lifespan)
async def _serve_sidecar(self, app: "Application") -> None:
sidecar_app = _build_sidecar_app(app)
config = uvicorn.Config(
sidecar_app,
host=self.sidecar_http_host,
port=self.sidecar_http_port,
log_level="warning",
access_log=False,
)
server = uvicorn.Server(config)
self._publish_sidecar_info()
self.logger.info(
f"MCP HTTP sidecar listening on " f"http://{self.sidecar_http_host}:{self.sidecar_http_port}",
)
await server.serve()
def _publish_sidecar_info(self) -> None:
if not self.sidecar_info_path:
return
info_path = Path(self.sidecar_info_path)
info_path.parent.mkdir(parents=True, exist_ok=True)
info_path.write_text(
json.dumps({"host": self.sidecar_http_host, "port": self.sidecar_http_port}),
encoding="utf-8",
)
def _cleanup_sidecar_info(self) -> None:
if not self.sidecar_info_path:
return
try:
Path(self.sidecar_info_path).unlink(missing_ok=True)
except Exception:
pass
def add_job(self, job: "BaseJob") -> None:
if isinstance(job, StreamJob):
return

35
reme2/config/config_parser.py
View file

@ -1,6 +1,8 @@
"""Parser for YAML config with CLI argument overrides."""
import json
import os
import re
from pathlib import Path
from typing import Any
@ -9,6 +11,32 @@ import yaml
# Config files are looked up relative to this module's directory
_CONFIG_DIR = Path(__file__).parent
_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
def _expand_env_vars(value: Any) -> Any:
"""Recursively expand `${VAR}` / `${VAR:-default}` placeholders in strings.
Lets vault.yaml reference secrets / per-host settings without baking
them into the file. Unset vars without a default raise ValueError so
typos don't silently produce empty connection strings.
"""
if isinstance(value, str):
def repl(m: re.Match) -> str:
name = m.group(1)
default = m.group(2)
v = os.environ.get(name)
if v is None:
if default is not None:
return default
raise ValueError(f"Config references undefined env var: {name}")
return v
return _ENV_VAR_RE.sub(repl, value)
if isinstance(value, dict):
return {k: _expand_env_vars(v) for k, v in value.items()}
if isinstance(value, list):
return [_expand_env_vars(v) for v in value]
return value
# Pre-scan config directory: maps basename(without ext) -> Path
@ -91,14 +119,15 @@ def _load_yaml(name_or_path: str, encoding: str = "utf-8") -> dict:
def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
"""Read YAML or JSON file based on extension."""
"""Read YAML or JSON file based on extension. Expands ${ENV_VAR}."""
with path.open(encoding=encoding) as f:
if path.suffix == ".json":
result = json.load(f)
return result if result is not None else {}
else:
result = yaml.safe_load(f)
return result if result is not None else {}
if result is None:
return {}
return _expand_env_vars(result)
def _deep_merge(base: dict, update: dict) -> dict:

220
reme2/config/curated.yaml Normal file
View file

@ -0,0 +1,220 @@
app_name: reme-curated
enable_logo: false
log_to_console: true
log_to_file: false
# Curated profile: three MCP tools surfaced —
# `query` — graph-aware hybrid retrieval
# `sync` — hot-path event sync (idempotent upsert per (date, name))
# `ingest` — LLM-driven distillation over existing files (cold path)
#
# The agent calls `sync` continuously through the task — picking a
# stable `name` per logical thread so each call extends the same
# event folder rather than fragmenting. At task completion (or
# PreCompact / SessionEnd) it calls `ingest` once to distill the
# active events into topic-level cognition.
#
# For full direct control over every memory_* primitive see ./full.yaml.
service:
backend: mcp
transport: stdio
sidecar_http: true
sidecar_http_host: "127.0.0.1"
sidecar_http_port: 8765
sidecar_info_path: "./vault/.reme/sidecar.json"
jobs:
- backend: base
name: query
description: |
Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink
BFS fusion. Use for "what do I know about X" / "did I work on Y" /
"what's connected to [[Z]]". Anchor mode: include `[[Target]]` in
the query to seed BFS at that file. Topic-rooted mode: pass `seeds`
explicitly. Returns chunks ranked by combined relevance + graph
proximity, each tagged with `graph_hop`.
parameters:
type: object
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.0 }
graph_depth:
type: integer
default: 1
description: "BFS hops from seeds. 1 covers immediate neighbors."
seeds:
type: array
items: { type: string }
description: "Explicit seed paths (topic-rooted mode)."
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_graph_search
- backend: base
name: sync
description: |
Hot-path event sync: idempotent upsert of an event FOLDER under
`events/{date}/{name}/`. The folder contains the index `{name}.md`
(Event schema, frontmatter + narrative + Materials footer) plus
any raw materials you pass — conversation snippets, tool outputs,
data dumps. The watcher indexes everything inside.
CONTINUITY MODEL: pick a stable `name` per logical thread and
call `sync` repeatedly through the task. Each call extends the
same folder:
* new `content` → appended under a `## Update — {iso}` section
* new `materials` → siblings (auto-suffix on filename collision)
* `topics` + `tags` merged (union) into frontmatter
* Materials footer regenerated to list every artifact in the folder
First call (folder doesn't exist) → CREATE; subsequent calls with
the same `name` while the event is `status: active` → APPEND.
If the event is `status: distilled` / `archived`, `sync` REFUSES
and returns a `suggested_name` so you start a fresh thread instead
of mutating prior cognition.
Zero LLM cost. Call CONTINUOUSLY through a task as facts land,
and especially at PreCompact to dump verbose raw text into
`materials` before context truncation loses it. The folder is
the unit `ingest` later reads from.
parameters:
type: object
required: [name]
properties:
name:
type: string
description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" }
content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." }
topics:
type: array
items: { type: string }
description: "related topic wikilinks. Unioned into frontmatter on append."
tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." }
materials:
type: array
description: "Raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision with an existing artifact auto-suffixes (foo.txt → foo-2.txt)."
items:
type: object
required: [filename, content]
properties:
filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" }
content: { type: string }
on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" }
origin_session_id: { type: string, description: "set on initial create only" }
steps:
- backend: sync
- backend: base
name: ingest
description: |
Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only:
task completion / SessionEnd / when the agent decides the working
set is ready. Not a per-turn tool.
The agent hands off the working set in two interchangeable forms
(use both as appropriate):
* `content` — inline material to distill: a hint / summary, or
raw text the agent is feeding directly.
* `related_paths` — paths the agent points at: event folder
indexes (Ingestor follows `## Materials` to read each
artifact), individual material files, or candidate topics.
The Ingestor reads the working set + linked topics, decides which
existing topics to update / create, and flips each distilled
event's status to "distilled". Returns an audit trail.
NOT for raw event logging — that's `sync`'s job.
parameters:
type: object
required: [content]
properties:
content:
type: string
description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set."
hint:
type: string
description: "Caller guidance about target / intent."
target_path:
type: string
description: "Optional suggested path; required for the no-LLM degraded path."
metadata:
type: object
description: "Suggested frontmatter for any new topic."
related_paths:
type: array
items: { type: string }
description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics."
steps:
- backend: ingestor
components:
# Ingestor LLM (opt-in). Without this the Ingestor degrades to a
# direct create from explicit `target_path`; edits/renames/deletes
# require the LLM. Uncomment + provide LLM_API_KEY to enable.
#
# as_llm:
# default:
# backend: openai
# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
# api_key: ${LLM_API_KEY}
# client_kwargs:
# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
# stream: false
#
# as_llm_formatter:
# default:
# backend: openai
as_token_counter:
default:
backend: estimated
# Embedding is opt-in: leave embedding_model="" on file_store to run
# keyword-only; uncomment + flip to "default" to enable hybrid search.
#
# embedding_model:
# default:
# backend: openai
# model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
# dimensions: 1536
# pass_dimensions: false
# enable_cache: true
# max_batch_size: 10
# max_cache_size: 2000
# max_input_length: 8192
edge_extractor:
default:
backend: regex
file_parser:
md:
backend: md
edge_extractor: default
default:
backend: text
file_store:
default:
backend: local
embedding_model: ""
store_name: "reme"
db_path: "./vault/.reme"
fts_enabled: true
file_watcher:
default:
backend: full
file_store: default
default_parser: md
watch_path: "./vault"
recursive: true
# Retriever (`hybrid`) is a Step, not a pre-instantiated component —
# the `query` shell builds it on demand. Tune defaults by attaching
# knobs to the `memory_graph_search` step under the `query` job above.

409
reme2/config/full.yaml Normal file
View file

@ -0,0 +1,409 @@
app_name: reme-full
enable_logo: false
log_to_console: true
log_to_file: false
# Full-exposure profile: every memory_* read + write primitive + the
# typed `topic_create` + the LLM-driven `ingest` are surfaced as MCP
# tools. The agent picks whatever it needs — finest granularity, no
# opinion enforced beyond the wikilink-uniqueness gate.
#
# Use when you want the agent to manage memory directly with full
# control. For an opinionated minimal surface, see ./curated.yaml
# (only `query` + `ingest`).
service:
backend: mcp
transport: stdio
sidecar_http: true
sidecar_http_host: "127.0.0.1"
sidecar_http_port: 8765
sidecar_info_path: "./vault/.reme/sidecar.json"
jobs:
# -- Write entry points ------------------------------------------------
- backend: base
name: sync
description: |
Hot-path event sync: idempotent upsert of an event FOLDER under
`events/{date}/{name}/`. The folder contains the index `{name}.md`
(Event schema, frontmatter + narrative + Materials footer) plus
any raw materials you pass — conversation snippets, tool outputs,
data dumps. The watcher indexes everything inside.
CONTINUITY MODEL: pick a stable `name` per logical thread and
call `sync` repeatedly through the task. Each call extends the
same folder:
* new `content` → appended under a `## Update — {iso}` section
* new `materials` → siblings (auto-suffix on filename collision)
* `topics` + `tags` merged (union) into frontmatter
* Materials footer regenerated to list every artifact in the folder
First call (folder doesn't exist) → CREATE; subsequent calls with
the same `name` while the event is `status: active` → APPEND.
If the event is `status: distilled` / `archived`, `sync` REFUSES
and returns a `suggested_name` so you start a fresh thread
instead of mutating prior cognition.
Zero LLM cost. Call CONTINUOUSLY through a task as facts land,
and especially at PreCompact to dump verbose raw text into
`materials` before context truncation loses it. The folder is
the unit `ingest` later reads from.
parameters:
type: object
required: [name]
properties:
name:
type: string
description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" }
content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." }
topics:
type: array
items: { type: string }
description: "related topic wikilinks ('[[X]]' or '[[topics/X/X]]'). Unioned into frontmatter on append."
tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." }
materials:
type: array
description: "Raw artifacts written as siblings of the index inside the event folder. Filenames must be safe (letters/digits/dot/underscore/dash). On filename collision with an existing artifact, auto-suffixes (foo.txt → foo-2.txt)."
items:
type: object
required: [filename, content]
properties:
filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt', 'snapshot.json'" }
content: { type: string }
on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" }
origin_session_id: { type: string, description: "set on initial create only" }
steps:
- backend: sync
- backend: base
name: ingest
description: |
Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only:
task completion / SessionEnd / when the agent decides the working
set is ready. Not a per-turn tool.
The agent hands off the working set in two interchangeable forms
(use both as appropriate):
* `content` — inline material to distill: a hint / summary, or
raw text the agent is feeding directly.
* `related_paths` — paths the agent points at: event folder
indexes (the Ingestor will follow `## Materials` and read
each artifact), individual material files, or candidate
topics flagged for update.
The Ingestor reads the working set + linked topics, decides which
existing topics to update / create, and flips each distilled
event's status to "distilled".
NOT for raw event logging — use `sync` for that.
parameters:
type: object
required: [content]
properties:
content:
type: string
description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set."
hint: { type: string, description: "Caller guidance about target / intent." }
target_path: { type: string, description: "Optional suggested path; required for the no-LLM degraded path." }
metadata: { type: object, description: "Suggested frontmatter for any new topic." }
related_paths:
type: array
items: { type: string }
description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor will read materials from `## Materials`), individual material files, or candidate topics."
steps:
- backend: ingestor
- backend: base
name: topic_create
description: |
Create a typed topic at `topics/{folder}/{name}.md` with hard
schema validation (judgment categories require `confidence`) and
wikilink-uniqueness enforcement.
parameters:
type: object
required: [folder, name, category]
properties:
folder: { type: string }
name: { type: string }
category:
type: string
enum: [company, sector, concept, method, tool, profile, thesis, model, questions, fundamentals]
description: { type: string }
content: { type: string }
confidence: { type: string, enum: ["⏳", "✅", "❌"] }
market: { type: string }
ticker: { type: string }
tags: { type: array, items: { type: string } }
steps:
- backend: topic_create
# -- Read tools --------------------------------------------------------
- backend: base
name: memory_search
description: "Hybrid (vector + keyword) search over chunks."
parameters:
type: object
required: [query]
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.1 }
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_search
- backend: base
name: memory_graph_search
description: |
Three-way fusion search: vector + keyword + graph (BFS over wikilinks).
Pulls in chunks reachable through linked topics/events that pure
relevance search would miss.
parameters:
type: object
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.0 }
graph_depth: { type: integer, default: 1 }
seeds: { type: array, items: { type: string } }
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_graph_search
- backend: base
name: memory_get
description: "Read frontmatter + body of a single file."
parameters:
type: object
required: [path]
properties:
path: { type: string }
include_chunks: { type: boolean, default: false }
steps:
- backend: memory_get
- backend: base
name: memory_list
description: |
List indexed files filtered by frontmatter exact-match, tags, or
path prefix. Returns {items: [{path, metadata}], count}.
parameters:
type: object
properties:
metadata: { type: object }
tags: { type: array, items: { type: string } }
path_prefix: { type: string }
limit: { type: integer, default: 100 }
steps:
- backend: memory_list
- backend: base
name: memory_backlinks
description: "Files linking TO the given path (with edge predicates)."
parameters:
type: object
required: [path]
properties:
path: { type: string }
steps:
- backend: memory_backlinks
- backend: base
name: memory_links
description: "Files the given path links to (resolved, with edge predicates)."
parameters:
type: object
required: [path]
properties:
path: { type: string }
steps:
- backend: memory_links
- backend: base
name: memory_resolve_wikilink
description: |
Resolve a `[[target]]` wikilink to a vault path. Stem-form (`X`)
consults the file_store's stem index; path-form (`a/b` or `a/b.md`)
is anchored at the vault root.
parameters:
type: object
required: [wikilink]
properties:
wikilink: { type: string }
steps:
- backend: memory_resolve_wikilink
- backend: base
name: memory_count_tokens
description: "Estimate token count for a file body or raw text."
parameters:
type: object
properties:
path: { type: string }
text: { type: string }
steps:
- backend: memory_count_tokens
# -- Write primitives (raw building blocks) ----------------------------
- backend: base
name: memory_create
description: |
Create a new file (raw primitive — no LLM reasoning). Prefer
`ingest` when you want an LLM curator. Wikilink-uniqueness gate
runs unless `force=true`.
parameters:
type: object
required: [path]
properties:
path: { type: string }
metadata: { type: object }
content: { type: string }
overwrite: { type: boolean, default: false }
force: { type: boolean, default: false }
steps:
- backend: memory_create
- backend: base
name: memory_update
description: |
Edit-style content update: replace `old_string` with `new_string`
in the file body. For frontmatter changes use `memory_property_update`.
parameters:
type: object
required: [path, old_string, new_string]
properties:
path: { type: string }
old_string: { type: string }
new_string: { type: string }
replace_all: { type: boolean, default: false }
steps:
- backend: memory_update
- backend: base
name: memory_property_update
description: "Update a single YAML frontmatter key. value=null deletes the key."
parameters:
type: object
required: [path, key]
properties:
path: { type: string }
key: { type: string }
value: {}
steps:
- backend: memory_property_update
- backend: base
name: memory_rename
description: |
Rename a file and rewrite all incoming `[[wikilink]]` references
across the vault. Refuses on destination conflict or stem ambiguity.
parameters:
type: object
required: [old_path, new_path]
properties:
old_path: { type: string }
new_path: { type: string }
steps:
- backend: memory_rename
- backend: base
name: memory_delete
description: "Delete a file."
parameters:
type: object
required: [path]
properties:
path: { type: string }
steps:
- backend: memory_delete
- backend: base
name: memory_archive
description: |
Archive a file: flip `status: archived` then move under
`<vault>/<archive_dir>/<original_relative_path>`.
parameters:
type: object
required: [path]
properties:
path: { type: string }
archive_dir: { type: string, default: "Archive" }
steps:
- backend: memory_archive
components:
# Ingestor LLM (opt-in). Without this the Ingestor degrades to a
# direct create from explicit `target_path`; edits/renames/deletes
# require the LLM. Uncomment + provide LLM_API_KEY to enable.
#
# as_llm:
# default:
# backend: openai
# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
# api_key: ${LLM_API_KEY}
# client_kwargs:
# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
# stream: false
#
# as_llm_formatter:
# default:
# backend: openai
as_token_counter:
default:
backend: estimated
# Embedding is opt-in: leave embedding_model="" on file_store to run
# keyword-only; uncomment + flip to "default" to enable hybrid search.
#
# embedding_model:
# default:
# backend: openai
# model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
# dimensions: 1536
# pass_dimensions: false
# enable_cache: true
# max_batch_size: 10
# max_cache_size: 2000
# max_input_length: 8192
edge_extractor:
default:
backend: regex
file_parser:
md:
backend: md
edge_extractor: default
default:
backend: text
file_store:
default:
backend: local
embedding_model: ""
store_name: "reme"
db_path: "./vault/.reme"
fts_enabled: true
file_watcher:
default:
backend: full
file_store: default
default_parser: md
watch_path: "./vault"
recursive: true
# Retriever (`hybrid`) is a Step, not a pre-instantiated component —
# the `memory_search` / `memory_graph_search` shells build it on
# demand. To tune defaults, pass knobs (`vector_weight`, `graph_*`,
# …) on the step config under each job below.

21
reme2/config/paw.yaml
View file

@ -42,13 +42,28 @@ components:
max_cache_size: 2000
max_input_length: 8192
edge_extractor:
default:
backend: regex
llm:
backend: llm
as_llm: default
as_llm_formatter: default
file_store: default
max_input_chars: 8000
min_confidence: 0.5
max_iters: 8
file_parser:
md:
backend: md
edge_extractor: default
embedding_model: default
default:
backend: default
backend: text
embedding_model: default
chunk_store:
file_store:
default:
backend: local
embedding_model: default
@ -58,7 +73,7 @@ components:
file_watcher:
default:
backend: full
chunk_store: default
file_store: default
default_parser: default
watch_paths: [ "./test_data" ]
recursive: true

308
reme2/config/smoke_test.py Normal file
View file

@ -0,0 +1,308 @@
"""Smoke-test reme2/config/full.yaml + reme2/config/curated.yaml.
Boots Application with each profile against a temporary vault, lists
the registered jobs, then calls a representative subset to confirm
end-to-end wiring (parser → file_store → memory_io / ingest /
memory_graph_search). The Ingestor degrades gracefully when no LLM is
configured — we exercise the degraded path so the test is hermetic.
Run:
python reme2/config/smoke_test.py
"""
import asyncio
import json
import sys
import tempfile
from pathlib import Path
_REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(_REPO_ROOT))
# Eager-import side-effect modules so all @R.register() decorators run.
import reme2 # noqa: E402,F401
import reme2.mcp.steps.memory_io # noqa: E402,F401
import reme2.mcp.steps.memory_retriever # noqa: E402,F401
import reme2.memory.ingestor # noqa: E402,F401
import reme2.memory.summarizer # noqa: E402,F401
import reme2.memory.maintainer # noqa: E402,F401
import reme2.mcp.steps # noqa: E402,F401
from reme2.application import Application # noqa: E402
from reme2.config import parse_args # noqa: E402
CONFIG_DIR = Path(__file__).resolve().parent
PROFILES = {
"full": CONFIG_DIR / "full.yaml",
"curated": CONFIG_DIR / "curated.yaml",
}
def _seed_vault(vault: Path) -> None:
"""Drop a few markdown files so reads return something."""
(vault / "topics" / "Alice").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Alice" / "Alice.md").write_text(
"---\ntitle: Alice\ncategory: profile\ntags: [person]\n---\n"
"# Alice\n\nAlice works on [[Project X]] with [[Bob]].\n"
"[author:: [[Alice]]]\n",
encoding="utf-8",
)
(vault / "topics" / "Bob").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Bob" / "Bob.md").write_text(
"---\ntitle: Bob\ncategory: profile\ntags: [person]\n---\n"
"# Bob\n\nBob collaborates with [[Alice]] on [[Project X]].\n",
encoding="utf-8",
)
(vault / "topics" / "Project X").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Project X" / "Project X.md").write_text(
"---\ntitle: Project X\ncategory: concept\n---\n"
"# Project X\n\nA major initiative led by [[Alice]] with [[Bob]].\n",
encoding="utf-8",
)
async def _wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> None:
last = -1
stable_for = 0
for _ in range(int(timeout_s / 0.25)):
now = len(watcher.file_store)
if now == last and now >= expected_min:
stable_for += 1
if stable_for >= 4:
return
else:
stable_for = 0
last = now
await asyncio.sleep(0.25)
def _decode(resp) -> object:
"""Job answers are JSON strings; decode for inspection. Pass through dicts/lists."""
if isinstance(resp.answer, (dict, list)):
return resp.answer
if isinstance(resp.answer, str):
try:
return json.loads(resp.answer)
except (json.JSONDecodeError, TypeError):
return resp.answer
return resp.answer
async def _run_profile(name: str, config_path: Path) -> dict:
"""Boot the profile against a fresh temp vault; exercise jobs; return summary."""
print(f"\n========== profile: {name} ==========")
with tempfile.TemporaryDirectory() as tmp:
vault = Path(tmp) / "vault"
vault.mkdir()
_seed_vault(vault)
# Override watch_path / db_path / sidecar info; force HTTP service so
# we never block on stdio MCP (we only call jobs directly).
_, cfg = parse_args(
"start",
f"config={config_path}",
f"components.file_watcher.default.watch_path={vault}",
f"components.file_store.default.db_path={vault}/.reme",
)
cfg["service"] = {"backend": "http"}
app = Application(**cfg)
await app.start()
try:
jobs = sorted(app.context.jobs.keys())
print(f" registered jobs ({len(jobs)}): {jobs}")
watcher = app.context.components["file_watcher"]["default"]
await _wait_for_index(watcher, expected_min=3)
print(f" file_store nodes after sync: {len(watcher.file_store)}")
assert len(watcher.file_store) >= 3, "watcher did not index seed files"
results: dict = {"jobs": jobs, "checks": []}
alice_path = str((vault / "topics" / "Alice" / "Alice.md").resolve())
if "memory_get" in jobs:
r = _decode(await app.run_job("memory_get", path=alice_path))
ok = isinstance(r, dict) and r.get("exists") is True
print(f" memory_get(Alice.md) → exists={ok}, edges={len(r.get('link', []))}")
results["checks"].append(("memory_get", ok))
if "memory_list" in jobs:
r = _decode(await app.run_job("memory_list", tags=["person"]))
ok = isinstance(r, dict) and r.get("count", 0) >= 2
print(f" memory_list(tags=[person]) → count={r.get('count') if isinstance(r, dict) else '?'}")
results["checks"].append(("memory_list", ok))
if "memory_search" in jobs:
r = _decode(await app.run_job("memory_search", query="collaborates", max_results=3, min_score=0.0))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
ok = len(hits) > 0
print(f" memory_search('collaborates') → {len(hits)} hits")
results["checks"].append(("memory_search", ok))
if "memory_links" in jobs:
r = _decode(await app.run_job("memory_links", path=alice_path))
ok = isinstance(r, dict) and len(r.get("links", [])) >= 2
print(f" memory_links(Alice.md) → {len(r.get('links', []))} resolved")
results["checks"].append(("memory_links", ok))
if "query" in jobs:
r = _decode(await app.run_job("query", query="Alice Bob", max_results=3, min_score=0.0))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
ok = len(hits) > 0
print(f" query('Alice Bob') → {len(hits)} hits")
results["checks"].append(("query", ok))
if "ingest" in jobs:
# Degraded path: no LLM key set in env → Ingestor falls back
# to direct create from `target_path`. That's the hermetic path.
target = str((vault / "topics" / "smoke" / "smoke.md").resolve())
r = _decode(await app.run_job(
"ingest",
content="# smoke topic\n\nrecorded by config smoke test.\n",
target_path=target,
metadata={"category": "concept", "title": "smoke"},
))
ok = isinstance(r, dict) and (r.get("applied") or r.get("skipped"))
print(f" ingest(degraded create) → applied={len(r.get('applied', [])) if isinstance(r, dict) else '?'}, "
f"used_llm={r.get('used_llm') if isinstance(r, dict) else '?'}")
results["checks"].append(("ingest", bool(ok)))
# Confirm the file landed on disk.
assert Path(target).is_file(), "ingest did not produce the target file"
if "sync" in jobs:
# CREATE call.
r = _decode(await app.run_job(
"sync",
name="smoke-event",
description="smoke test event",
content="## ops\n- ran the smoke test\n",
topics=["[[Alice]]"],
tags=["smoke"],
materials=[
{"filename": "raw-prompt.md", "content": "# raw user prompt\n\nrun the smoke test\n"},
{"filename": "tool-output.txt", "content": "tool ran ok\nexit=0\n"},
],
))
ok = (
isinstance(r, dict)
and r.get("created") is True
and r.get("action") == "created"
and len(r.get("materials", [])) == 2
)
materials = r.get("materials", []) if isinstance(r, dict) else []
print(f" sync(create smoke-event w/ 2 materials) → created={r.get('created') if isinstance(r, dict) else '?'}, "
f"materials={len(materials)}")
results["checks"].append(("sync.create", bool(ok)))
for m in materials:
assert Path(m).is_file(), f"event material missing: {m}"
if materials:
event_dir = Path(materials[0]).parent
index_text = (event_dir / "smoke-event.md").read_text(encoding="utf-8")
assert "## Materials" in index_text, "index .md missing Materials section"
assert "raw-prompt.md" in index_text, "Materials section missing raw-prompt link"
await _wait_for_index(watcher, expected_min=len(watcher.file_store) + 2)
# APPEND call: same name, new content + new + colliding material.
r2 = _decode(await app.run_job(
"sync",
name="smoke-event",
content="## follow-up\n- second pass facts\n",
topics=["[[Bob]]"], # union with prior [[Alice]]
tags=["follow-up"], # union with prior [smoke]
materials=[
{"filename": "tool-output.txt", "content": "second tool run\nexit=0\n"}, # collision → auto-suffix
{"filename": "summary.md", "content": "# summary\nsecond pass\n"},
],
))
ok2 = (
isinstance(r2, dict)
and r2.get("created") is False
and r2.get("action") == "appended"
and len(r2.get("materials", [])) == 2
)
appended_paths = r2.get("materials", []) if isinstance(r2, dict) else []
print(f" sync(append smoke-event w/ collision) → action={r2.get('action') if isinstance(r2, dict) else '?'}, "
f"new_materials={len(appended_paths)}")
results["checks"].append(("sync.append", bool(ok2)))
# Collision should have produced tool-output-2.txt; summary.md untouched.
names_appended = {Path(p).name for p in appended_paths}
assert "tool-output-2.txt" in names_appended, f"collision auto-suffix missing: {names_appended}"
assert "summary.md" in names_appended, f"clean filename missing: {names_appended}"
# Index should now contain BOTH the original ops section and the Update section.
if materials:
index_text2 = (Path(materials[0]).parent / "smoke-event.md").read_text(encoding="utf-8")
assert "## ops" in index_text2, "original content lost on append"
assert "## Update —" in index_text2, "missing Update section after append"
assert "follow-up" in index_text2, "appended content not in index"
# Frontmatter union check
assert "[[Bob]]" in index_text2, "topic union failed"
# Materials footer should now list 4 files (2 original + summary + tool-output-2)
assert "tool-output-2.txt" in index_text2, "Materials footer missing collided file"
assert "summary.md" in index_text2, "Materials footer missing new file"
# REFUSAL: flip status to distilled, third sync should refuse.
index_path = str(Path(materials[0]).parent / "smoke-event.md") if materials else None
if index_path and "memory_property_update" in jobs:
await app.run_job("memory_property_update", path=index_path, key="status", value="distilled")
r3 = _decode(await app.run_job(
"sync",
name="smoke-event",
content="should refuse",
))
ok3 = isinstance(r3, dict) and "error" in r3 and r3.get("status") == "distilled"
print(f" sync(refuse on distilled) → error={'error' in (r3 if isinstance(r3, dict) else {})}, "
f"suggested_name={r3.get('suggested_name') if isinstance(r3, dict) else '?'}")
results["checks"].append(("sync.refuse_distilled", bool(ok3)))
if "topic_create" in jobs:
r = _decode(await app.run_job(
"topic_create",
folder="Carol",
name="Carol",
category="profile",
description="smoke test",
content="# Carol\n",
tags=["person"],
))
ok = isinstance(r, dict) and (r.get("created") is True or "path" in r)
print(f" topic_create(Carol) → created={r.get('created') if isinstance(r, dict) else '?'}")
results["checks"].append(("topic_create", bool(ok)))
return results
finally:
await app.close()
async def _main() -> int:
summary: dict[str, dict] = {}
for name, path in PROFILES.items():
try:
summary[name] = await _run_profile(name, path)
except Exception as e:
print(f" ✗ profile '{name}' failed: {type(e).__name__}: {e}")
import traceback
traceback.print_exc()
summary[name] = {"error": str(e)}
print("\n========== summary ==========")
failed = 0
for name, result in summary.items():
if "error" in result:
print(f" {name}: ERROR — {result['error']}")
failed += 1
continue
checks = result.get("checks", [])
passed = sum(1 for _, ok in checks if ok)
total = len(checks)
marker = "✓" if passed == total else "✗"
print(f" {marker} {name}: {passed}/{total} checks passed; jobs={len(result['jobs'])}")
for label, ok in checks:
if not ok:
print(f" ✗ {label}")
failed += 1
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(asyncio.run(_main()))

4
reme2/enumeration/component_enum.py
View file

@ -24,9 +24,11 @@ class ComponentEnum(str, Enum):
EMBEDDING_MODEL = "embedding_model"
EDGE_EXTRACTOR = "edge_extractor"
FILE_PARSER = "file_parser"
CHUNK_STORE = "chunk_store"
FILE_STORE = "file_store"
FILE_WATCHER = "file_watcher"

17
reme2/file_based/__init__.py
View file

@ -1,17 +0,0 @@
"""File-based components and utilities."""
from .file_io import FileIO
from .file_utils import (
async_read_file_safe,
truncate_text_output,
)
from .memory_search import MemorySearch
from .summarizer import Summarizer
__all__ = [
"FileIO",
"async_read_file_safe",
"truncate_text_output",
"MemorySearch",
"Summarizer",
]

361
reme2/file_based/file_io.py
View file

@ -1,361 +0,0 @@
"""File I/O operations with a configurable working directory."""
import os
from pathlib import Path
import aiofiles
from agentscope.message import TextBlock
from agentscope.tool import ToolResponse
from .file_utils import async_read_file_safe, truncate_text_output
from ..constants import TRUNCATION_NOTICE_MARKER
class FileIO:
"""File I/O operations with a configurable working directory."""
def __init__(self, working_dir: str | Path):
"""Initialize FileIO with a working directory.
Args:
working_dir (`str`):
The working directory for resolving relative paths.
"""
self.working_dir = Path(working_dir)
def _resolve_file_path(self, file_path: str) -> str:
"""Resolve file path: use absolute path as-is,
resolve relative path from working_dir.
Args:
file_path: The input file path (absolute or relative).
Returns:
The resolved absolute file path as string.
"""
path = Path(file_path).expanduser()
if path.is_absolute():
return str(path)
else:
return str(self.working_dir / file_path)
async def read_file( # pylint: disable=too-many-return-statements
self,
file_path: str,
start_line: int | None = None,
end_line: int | None = None,
) -> ToolResponse:
"""Read a file. Relative paths resolve from WORKING_DIR.
Use start_line/end_line to read a specific line range (output includes
line numbers). Omit both to read the full file.
Args:
file_path (`str`):
Path to the file.
start_line (`int`, optional):
First line to read (1-based, inclusive).
end_line (`int`, optional):
Last line to read (1-based, inclusive).
"""
# Convert start_line/end_line to int if they are strings
if start_line is not None:
try:
start_line = int(start_line)
except (ValueError, TypeError):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: start_line must be an integer, got {start_line!r}.",
),
],
)
if end_line is not None:
try:
end_line = int(end_line)
except (ValueError, TypeError):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: end_line must be an integer, got {end_line!r}.",
),
],
)
file_path = self._resolve_file_path(file_path)
if not os.path.exists(file_path):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The file {file_path} does not exist.",
),
],
)
if not os.path.isfile(file_path):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The path {file_path} is not a file.",
),
],
)
try:
content = await async_read_file_safe(file_path)
all_lines = content.split("\n")
total = len(all_lines)
# Determine read range
s = max(1, start_line if start_line is not None else 1)
e = min(total, end_line if end_line is not None else total)
if s > total:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: start_line {s} exceeds file length ({total} lines).",
),
],
)
if s > e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: start_line ({s}) > end_line ({e}).",
),
],
)
# Extract selected lines
selected_content = "\n".join(all_lines[s - 1: e])
# Apply smart truncation (consistent with shell output format)
text = truncate_text_output(
selected_content,
start_line=s,
total_lines=total,
file_path=file_path,
)
# Add continuation hint if partial read without truncation.
# Use TRUNCATION_NOTICE_MARKER format so ToolResultCompactor can
# re-truncate with the correct start_line when compacting old messages.
if text == selected_content and e < total:
content_bytes = len(text.encode("utf-8"))
notice = (
TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated."
f"\nThe full content is saved to the file "
f"and contains {total} lines in total."
f"\nThis excerpt starts at line {s} and "
f"covers the next {content_bytes} bytes."
"\nIf the current content is not enough, "
f"call `read_file` with file_path={file_path} start_line={e + 1} to read more."
)
text = text + notice
return ToolResponse(
content=[TextBlock(type="text", text=text)],
)
except Exception as e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Read file failed due to \n{e}",
),
],
)
async def write_file(
self,
file_path: str,
content: str,
) -> ToolResponse:
"""Create or overwrite a file. Relative paths resolve from working_dir.
Args:
file_path (`str`):
Path to the file.
content (`str`):
Content to write.
"""
if not file_path:
return ToolResponse(
content=[
TextBlock(
type="text",
text="Error: No `file_path` provided.",
),
],
)
file_path = self._resolve_file_path(file_path)
try:
async with aiofiles.open(file_path, "w", encoding="utf-8") as file:
await file.write(content)
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Wrote {len(content)} bytes to {file_path}.",
),
],
)
except Exception as e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Write file failed due to \n{e}",
),
],
)
# pylint: disable=too-many-return-statements
async def edit_file(
self,
file_path: str,
old_text: str,
new_text: str,
) -> ToolResponse:
"""Find-and-replace text in a file. All occurrences of old_text are
replaced with new_text. Relative paths resolve from working_dir.
Args:
file_path (`str`):
Path to the file.
old_text (`str`):
Exact text to find.
new_text (`str`):
Replacement text.
"""
if not file_path:
return ToolResponse(
content=[
TextBlock(
type="text",
text="Error: No `file_path` provided.",
),
],
)
resolved_path = self._resolve_file_path(file_path)
if not os.path.exists(resolved_path):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The file {resolved_path} does not exist.",
),
],
)
if not os.path.isfile(resolved_path):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The path {resolved_path} is not a file.",
),
],
)
try:
content = await async_read_file_safe(resolved_path)
except Exception as e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Read file failed due to \n{e}",
),
],
)
if old_text not in content:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The text to replace was not found in {file_path}.",
),
],
)
new_content = content.replace(old_text, new_text)
write_response = await self.write_file(file_path=resolved_path, content=new_content)
if write_response.content and len(write_response.content) > 0:
write_text = write_response.content[0].get("text", "")
if write_text.startswith("Error:"):
return write_response
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Successfully replaced text in {file_path}.",
),
],
)
async def append_file(
self,
file_path: str,
content: str,
) -> ToolResponse:
"""Append content to the end of a file. Relative paths resolve from
working_dir.
Args:
file_path (`str`):
Path to the file.
content (`str`):
Content to append.
"""
if not file_path:
return ToolResponse(
content=[
TextBlock(
type="text",
text="Error: No `file_path` provided.",
),
],
)
file_path = self._resolve_file_path(file_path)
try:
async with aiofiles.open(file_path, "a", encoding="utf-8") as file:
await file.write(content)
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Appended {len(content)} bytes to {file_path}.",
),
],
)
except Exception as e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Append file failed due to \n{e}",
),
],
)

201
reme2/file_based/file_utils.py
View file

@ -1,201 +0,0 @@
"""Shared utilities for file and shell tools."""
import re
import aiofiles
from ..constants import (
DEFAULT_MAX_BYTES,
MAX_FILE_READ_BYTES,
TRUNCATION_NOTICE_MARKER,
)
def _truncate_fresh(
text: str,
start_line: int,
total_lines: int,
max_bytes: int,
file_path: str | None,
encoding: str,
) -> str:
"""Truncate fresh text (no prior truncation marker) by bytes with line integrity.
Slices at the byte boundary and appends a truncation notice with a continuation
hint so callers know which line to read next.
Returns the original text unchanged when it fits within max_bytes, or when the
last line itself exceeds max_bytes (unhandled edge case).
"""
text_bytes = text.encode(encoding)
# Under the byte limit — return as-is without any modification.
if len(text_bytes) <= max_bytes:
return text
# Slice at the byte boundary.
# Assuming every single line is shorter than DEFAULT_MAX_BYTES, this cut always
# lands mid-line, guaranteeing at least one complete line before the boundary.
# Lines that exceed DEFAULT_MAX_BYTES are not handled and may be skipped entirely.
truncated = text_bytes[:max_bytes]
# Decode back to str; errors="ignore" drops any split multibyte character
# at the cut boundary without raising an exception.
result = truncated.decode(encoding, errors="ignore")
# Count '\n' characters to determine how many complete lines are included.
# The tail after the final '\n' is a partial line that will be covered by
# the next read starting at next_line.
newline_count = result.count("\n")
# Compute the first line number not yet fully included in this chunk.
# max(1, ...) prevents next_line from equaling start_line when a single line
# exceeds max_bytes (newline_count == 0), which would make the caller retry
# the same range indefinitely.
next_line = start_line + max(1, newline_count)
if next_line <= total_lines:
# Truncation fell before the last line — continue reading from next_line.
read_from = next_line
elif start_line < total_lines:
# next_line overshot total_lines, meaning the cut landed inside the last line.
# Re-read from the start of the last line so the caller gets it in full.
read_from = total_lines
else:
# start_line == total_lines: the last line itself exceeds DEFAULT_MAX_BYTES.
# This case is outside our handled range — return without a truncation notice.
return result
notice = (
TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated."
f"\nThe full content is saved to the file and contains {total_lines} lines in total."
f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes."
f"\nIf the current content is not enough, call `read_file` with file_path={file_path or ''} "
f"start_line={read_from} to read more."
)
return result + notice
def _retruncate(
text: str,
max_bytes: int,
encoding: str,
) -> str:
"""Re-truncate text that was previously truncated (contains TRUNCATION_NOTICE_MARKER).
Extracts the original content before the marker, applies the new byte limit, and
updates the embedded notice (byte count and continuation line number) via regex.
Returns the original text unchanged when:
- the content already fits within max_bytes (with a small slack);
- required metadata fields cannot be parsed from the existing notice.
"""
parts = text.split(TRUNCATION_NOTICE_MARKER, 1)
original_content = parts[0]
old_notice = parts[1]
text_bytes = original_content.encode(encoding)
# Allow a small slack to avoid unnecessary re-truncation when content is just
# barely over the limit (e.g. due to minor encoding differences).
if len(text_bytes) <= max_bytes + 100:
return text
# Parse start_line from notice; return text unchanged if not found
start_match = re.search(r"starts at line (\d+)", old_notice)
if not start_match:
return text
start_line_parsed = int(start_match.group(1))
# Re-slice to the new byte limit.
# Because every line is assumed to be shorter than DEFAULT_MAX_BYTES, the cut
# always falls somewhere mid-line, so at least one complete line is preserved.
truncated_bytes = text_bytes[:max_bytes]
# errors="ignore" silently drops any incomplete multibyte character at the cut boundary.
result = truncated_bytes.decode(encoding, errors="ignore")
# Each '\n' in result corresponds to one fully-included line;
# anything after the last '\n' is a partial line that was cut off.
newline_count = result.count("\n")
# The next read should start at the line immediately after all complete lines.
# max(1, ...) guards against the theoretical zero-newline case
# (impossible when every line is shorter than DEFAULT_MAX_BYTES).
next_line = start_line_parsed + max(1, newline_count)
if not re.search(r"covers the next \d+ bytes", old_notice):
return text
# _truncate_fresh always includes a continuation hint, so both fields are always present.
new_notice = re.sub(r"covers the next \d+ bytes", f"covers the next {max_bytes} bytes", old_notice)
new_notice = re.sub(r"start_line=\d+ to read more", f"start_line={next_line} to read more", new_notice)
return result + TRUNCATION_NOTICE_MARKER + new_notice
def truncate_text_output(
text: str,
start_line: int = 1,
total_lines: int = 0,
max_bytes: int = DEFAULT_MAX_BYTES,
file_path: str | None = None,
encoding: str = "utf-8",
) -> str:
"""Truncate file output by bytes with line integrity.
If text is under byte limit, return as-is.
If over limit, truncate at the last complete line that fits,
allowing the next read to start from a fresh line.
Dispatches to :func:`_truncate_fresh` for text seen for the first time, or to
:func:`_retruncate` when the text already contains a TRUNCATION_NOTICE_MARKER
from a previous pass.
Args:
text: The output text to truncate.
start_line: The starting line number (1-based). Ignored when text already
contains a truncation notice (values are parsed from the notice instead).
total_lines: Total lines in the original file. Ignored when text already
contains a truncation notice (values are parsed from the notice instead).
max_bytes: Maximum size in bytes.
file_path: Optional file path to include in the truncation notice.
encoding: Character encoding used for byte-length calculation and decoding.
Returns:
Truncated text with notice if truncated.
"""
if not text:
return text
if max_bytes <= 0:
return text
try:
if TRUNCATION_NOTICE_MARKER in text:
return _retruncate(text, max_bytes=max_bytes, encoding=encoding)
else:
return _truncate_fresh(
text,
start_line=start_line,
total_lines=total_lines,
max_bytes=max_bytes,
file_path=file_path,
encoding=encoding,
)
except Exception:
return text
async def async_read_file_safe(file_path: str, max_bytes: int = MAX_FILE_READ_BYTES) -> str:
"""Async version of read_file_safe with Unicode error handling and memory protection.
Args:
file_path: Path to the file.
max_bytes: Maximum bytes to read into memory (default 1GB).
Returns:
File content as string (up to max_bytes).
"""
try:
async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
return await f.read(max_bytes)
except UnicodeDecodeError:
async with aiofiles.open(file_path, "r", encoding="utf-8", errors="ignore") as f:
return await f.read(max_bytes)

56
reme2/file_based/memory_search.py
View file

@ -1,56 +0,0 @@
"""Memory search step for semantic search in memory files."""
import json
from ..component import R
from ..component.base_step import BaseStep
@R.register("memory_search")
class MemorySearch(BaseStep):
"""Semantically search MEMORY.md and memory files."""
def __init__(self, vector_weight: float = 0.7, candidate_multiplier: float = 3.0, **kwargs):
"""Initialize memory search step.
Args:
vector_weight: Weight for vector search vs keyword search.
candidate_multiplier: Multiplier for candidate count before filtering.
**kwargs: Additional arguments passed to BaseStep.
"""
super().__init__(**kwargs)
self.vector_weight = vector_weight
self.candidate_multiplier = candidate_multiplier
async def execute(self):
"""Execute the memory search operation."""
assert self.context is not None, "Context is not set"
query: str = self.context.get("query", "").strip()
min_score: float = self.context.get("min_score", 0.1)
max_results: int = self.context.get("max_results", 5)
assert query, "Query cannot be empty"
assert (
isinstance(min_score, float | int) and 0.0 <= min_score <= 1.0
), f"min_score must be between 0 and 1, got {min_score}"
assert (
isinstance(max_results, int) and max_results > 0
), f"max_results must be a positive integer, got {max_results}"
chunk_filter = self.file_graph.filter(
paths=self.context.get("paths") or None,
tags=self.context.get("tags") or None,
exclude_paths=self.context.get("exclude_paths") or None,
)
results = await self.chunk_store.hybrid_search(
query=query,
limit=max_results,
vector_weight=self.vector_weight,
candidate_multiplier=self.candidate_multiplier,
chunk_filter=chunk_filter,
)
results = [r for r in results if r.score >= min_score]
return json.dumps([result.model_dump(exclude_none=True) for result in results], indent=2, ensure_ascii=False)

73
reme2/mcp/README.md Normal file
View file

@ -0,0 +1,73 @@
# reme2.mcp
Agent-facing MCP interface layer. Exposes the markdown vault under
`reme2/` (file_store + watcher + memory services) as MCP tools for
`claude-code` and other MCP clients.
## Layout
```
reme2/mcp/
├── __init__.py
├── server.py MCP server bootstrap; defaults to ../config/full.yaml
└── steps/ @R.register MCP step shells
├── memory_io.py memory_create/update/get/list/links/...
├── memory_retriever.py memory_search + memory_graph_search
├── sync.py hot-path event-folder upsert
└── topic_create.py typed topic creation w/ schema gates
```
The MCP layer's job is to **project** existing primitives as MCP tools
— it owns no business logic. Everything else lives outside the
transport boundary so memory services don't form an import cycle:
| Concern | Location |
|---|---|
| Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` |
| Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer |
| Pure write helpers + agent toolkit | `reme2/memory/memory_io.py` |
| Vault domain models (Event, Topic) | `reme2/schema/vault/` |
| Path templates + name disambiguation | `reme2/utils/vault_paths.py` |
| Step response serialization | `reme2/component/runtime_response.py` |
Dependency direction is strict:
```
reme2.mcp → reme2.memory → reme2.component / reme2.schema / reme2.utils
```
## Run
```bash
# stdio MCP server, full tool surface
python -m reme2.mcp.server
# override config or any field
python -m reme2.mcp.server config=reme2/config/curated.yaml
python -m reme2.mcp.server components.file_watcher.default.watch_path=/abs/vault
```
Config profiles (in `reme2/config/`):
- `full.yaml` — every memory_* primitive + sync + topic_create + ingest
- `curated.yaml` — opinionated 3-tool surface (`query`, `sync`, `ingest`)
## Tools exposed (full profile)
| Tool | Path | Purpose |
|---|---|---|
| `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation. |
| `topic_create` | steps/topic_create.py | Typed topic creation with schema gates. |
| `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. |
| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | steps/memory_io.py | Read primitives. |
| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | steps/memory_io.py | Raw write primitives (prefer `ingest` / `sync`). |
| `memory_count_tokens` | steps/memory_io.py | Token estimation. |
## Smoke test
```bash
python reme2/config/smoke_test.py
```
Boots both `full` and `curated` profiles in a temp vault, exercises a
representative subset of jobs end-to-end (in-process — no MCP
transport).

20
reme2/mcp/__init__.py Normal file
View file

@ -0,0 +1,20 @@
"""reme2.mcp — MCP interface layer.
The Agent-facing surface: server entrypoint + step shells that wrap
the three services in `reme2.memory` (Retriever, Ingestor, Maintainer)
plus hot-write primitives (sync, topic_create, memory_*) that bypass
services and land directly on the Memory File System.
This package depends on `reme2.memory`, `reme2.schema.vault`,
`reme2.utils`, `reme2.component` — never the reverse. Domain types
(Topic / Event), pure helpers (path builders, naming), and the write
primitives all live outside `mcp/` so memory-layer services can use
them without forming an import cycle through the transport layer.
Sub-packages:
steps/ - all @R.register MCP step shells (memory_io, memory_retriever,
sync, topic_create).
server.py - MCP server bootstrap (defaults to ../config/full.yaml).
"""
__version__ = "0.1.0"

74
reme2/mcp/server.py Normal file
View file

@ -0,0 +1,74 @@
"""vault MCP server entry point.
Usage:
python -m reme2.mcp.server [config=path/to/yaml] [service.transport=stdio]
python reme2/mcp/server.py [config=path/to/yaml] [service.transport=stdio]
By default loads `reme2/config/full.yaml` and starts the
ReMe2 application with the MCP service registered.
Env vars:
VAULT_PATH — override vault watch path (overrides config + cli args).
"""
import os
import sys
from pathlib import Path
# Make this work whether invoked as `python -m reme2.mcp.server` (repo
# root already on sys.path) or `python reme2/mcp/server.py` (only this
# file's dir is on sys.path — without this, `import reme2.mcp.steps`
# would fail with ModuleNotFoundError).
_REPO_ROOT = Path(__file__).resolve().parents[2]
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
from dotenv import load_dotenv # noqa: E402
# Load .env from repo root before anything else touches env vars
# (embedding/llm clients read them at construction time).
load_dotenv(_REPO_ROOT / ".env")
# Eagerly import side-effect modules so all @R.register() decorators run
# before Application introspects the registry.
import reme2 # noqa: E402,F401
import reme2.mcp.steps # noqa: E402,F401
import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
import reme2.memory.ingestor # noqa: E402,F401
import reme2.memory.summarizer # noqa: E402,F401
import reme2.component.service.mcp_service # noqa: E402,F401
from reme2.application import Application # noqa: E402
from reme2.config import parse_args # noqa: E402
_DEFAULT_CONFIG = str(_REPO_ROOT / "reme2" / "config" / "full.yaml")
def main() -> None:
argv = list(sys.argv[1:])
if not argv or argv[0].startswith("config=") or "=" in argv[0]:
argv.insert(0, "start")
if not any(a.startswith("config=") for a in argv[1:]):
argv.insert(1, f"config={_DEFAULT_CONFIG}")
vault_path = os.environ.get("VAULT_PATH")
if vault_path:
argv.append(f"components.file_watcher.default.watch_path={vault_path}")
argv.append(f"components.file_store.default.db_path={vault_path}/.reme")
argv.append(f"service.sidecar_info_path={vault_path}/.reme/sidecar.json")
sidecar_port = os.environ.get("VAULT_HTTP_PORT")
if sidecar_port:
argv.append(f"service.sidecar_http_port={sidecar_port}")
action, config = parse_args(*argv)
if action != "start":
raise SystemExit("reme2.mcp.server only supports the 'start' action")
app = Application(**config)
app.run_app()
if __name__ == "__main__":
main()

31
reme2/mcp/steps/__init__.py Normal file
View file

@ -0,0 +1,31 @@
"""MCP step shells — the @R.register classes the model invokes as MCP tools.
Two groups:
Hot-write primitives that bypass services and land on MFS directly:
sync — idempotent event-folder upsert (event log)
topic_create — typed topic creation (hard schema gates)
memory_io — memory_create / update / property_update / rename /
delete / archive / get / list / links / backlinks /
resolve_wikilink / count_tokens
Service delegate:
memory_retriever — memory_search + memory_graph_search; thin
wrappers that delegate to the configured
Retriever component (`reme2.memory.retriever`).
The Ingestor's MCP face (`ingest`) is registered from
`reme2.memory.ingestor`, since it's a service step rather than an MFS
primitive. Importing this package triggers all the step registrations
hosted here.
"""
from . import memory_io # noqa: F401 -- triggers @R.register for memory_*
from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search
from .sync import Sync
from .topic_create import TopicCreate
__all__ = [
"Sync",
"TopicCreate",
]

208
reme2/mcp/steps/memory_io.py Normal file
View file

@ -0,0 +1,208 @@
"""MCP step shells over the Memory File System engine API.
Per `structure.md`, .md files are the SSOT and the engine surface lives
in `reme2.memory.memory_io` (CRUD writes + MFS reads + Projections).
This module only hosts the `@R.register("memory_*")` Step shells —
each one translates RuntimeContext ↔ JSON payload and delegates to the
matching engine API function.
Search ops live in `memory_retriever.py` (Retriever composes V/K/graph
projections with policy).
"""
from __future__ import annotations
from pathlib import Path
from ...component import R
from ...component.base_step import BaseStep
from ...component.runtime_response import _set_answer
from ...enumeration import ComponentEnum
from ...memory import memory_io
@R.register("memory_get")
class MemoryGet(BaseStep):
"""Read a single memory file (frontmatter + body, optional chunks)."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
include_chunks: bool = bool(self.context.get("include_chunks", False))
assert path, "path is required"
result = await memory_io.read_file(self.file_store, path, include_chunks=include_chunks)
_set_answer(self.context, result)
@R.register("memory_list")
class MemoryList(BaseStep):
"""List indexed files filtered by frontmatter fields, tags, or path prefix."""
async def execute(self):
assert self.context is not None
result = memory_io.list_files(
self.file_store,
path_prefix=self.context.get("path_prefix"),
tags=self.context.get("tags") or [],
metadata=self.context.get("metadata") or {},
limit=int(self.context.get("limit", 100)),
)
_set_answer(self.context, result)
@R.register("memory_backlinks")
class MemoryBacklinks(BaseStep):
"""Files linking to a given path. Each entry carries the typed-edge predicate."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
assert path, "path is required"
_set_answer(self.context, memory_io.backlinks_of(self.file_store, path))
@R.register("memory_links")
class MemoryLinks(BaseStep):
"""Files a given path links to (resolved). Each entry carries the typed-edge predicate."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
assert path, "path is required"
_set_answer(self.context, memory_io.links_of(self.file_store, path))
@R.register("memory_resolve_wikilink")
class MemoryResolveWikilink(BaseStep):
"""Resolve a `[[target]]` wikilink with full ambiguity context."""
async def execute(self):
assert self.context is not None
wikilink: str = self.context.get("wikilink", "") or ""
assert wikilink, "wikilink is required"
payload = memory_io.wikilink_lookup(self.file_store, wikilink)
self.context.response.success = bool(payload.get("exists"))
_set_answer(self.context, payload)
@R.register("memory_create")
class MemoryCreate(BaseStep):
"""Create a markdown file. Wikilink-uniqueness gate runs unless force=True."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
metadata: dict = dict(self.context.get("metadata", {}) or {})
content: str = self.context.get("content", "") or ""
overwrite: bool = bool(self.context.get("overwrite", False))
force: bool = bool(self.context.get("force", False))
assert path, "path is required"
target = Path(path)
ok, payload = memory_io.write_create(
self.file_store, target,
metadata=metadata, content=content,
overwrite=overwrite, force=force,
)
self.context.response.success = ok
if ok:
payload = {**payload, "path": str(target.resolve())}
_set_answer(self.context, payload)
@R.register("memory_delete")
class MemoryDelete(BaseStep):
"""Delete a file. Watcher removes from store + graph."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
assert path, "path is required"
ok, payload = memory_io.write_delete(path)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_rename")
class MemoryRename(BaseStep):
"""Rename a file and rewrite incoming wikilinks across the vault."""
async def execute(self):
assert self.context is not None
old_path: str = self.context.get("old_path", "")
new_path: str = self.context.get("new_path", "")
assert old_path and new_path, "old_path and new_path are required"
watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[index,union-attr]
vault_root = Path(watcher.watch_path).resolve() # type: ignore[union-attr]
ok, payload = memory_io.write_rename(self.file_store, vault_root, old_path, new_path)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_property_update")
class MemoryPropertyUpdate(BaseStep):
"""Update a single YAML frontmatter key. value=null deletes the key."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
key: str = self.context.get("key", "")
value = self.context.get("value")
assert path and key, "path and key are required"
ok, payload = memory_io.write_property_update(path, key, value)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_update")
class MemoryUpdate(BaseStep):
"""Edit-style content update: replace `old_string` with `new_string`."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
old_string: str = self.context.get("old_string", "")
new_string: str = self.context.get("new_string", "")
replace_all: bool = bool(self.context.get("replace_all", False))
assert path, "path is required"
ok, payload = memory_io.write_update(path, old_string, new_string, replace_all=replace_all)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_archive")
class MemoryArchive(BaseStep):
"""Archive a file: flip `status: archived` then move to `<vault>/Archive/`."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
archive_dir_name: str = self.context.get("archive_dir", "Archive") or "Archive"
assert path, "path is required"
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
vault_root = Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd()
ok, payload = memory_io.write_archive(vault_root, path, archive_dir_name)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_count_tokens")
class MemoryCountTokens(BaseStep):
"""Estimate tokens for a file body or raw text. One of `path`/`text` required."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
text: str = self.context.get("text", "") or ""
result = await memory_io.count_tokens(
self.as_token_counter,
path=path or None,
text=text or None,
)
self.context.response.success = "error" not in result
_set_answer(self.context, result)

199
reme2/mcp/steps/memory_retriever.py Normal file
View file

@ -0,0 +1,199 @@
"""Memory retriever steps — thin MCP-facing wrappers over `BaseRetriever`.
Per the architecture blueprint, retrieval policy (V+K hybrid + graph
BFS fusion + ranking + intent routing) is the **Retriever service**
(`reme2.memory.retriever`), registered as `ComponentEnum.RETRIEVER`.
These steps are the MCP projection: they translate `RuntimeContext`
(paths/tags/exclude_paths filter, per-call knob overrides) into the
retriever's call surface, then serialize results for the API response
(joining file metadata onto each chunk so callers don't have to fire a
`memory_get` per hit).
Direct primary-key file ops (read/list/links/backlinks) live in
`memory_io` — those aren't retrieval.
"""
from __future__ import annotations
from ...component import R
from ...component.base_step import BaseStep
from ...component.runtime_response import _set_answer
from ...enumeration import ComponentEnum
from ...memory import memory_io
from ...memory.retriever import BaseRetriever, HybridRetriever
# Per-shell singleton cache: instantiating the retriever is cheap (it
# just stashes constructor knobs), but doing it once per call would
# still allocate every request. Keyed by step instance so each MCP
# job's overrides stay isolated.
_RETRIEVER_CACHE: dict[int, BaseRetriever] = {}
def _resolve_retriever(step: BaseStep) -> BaseRetriever:
"""Get (or build) the retriever instance for this MCP step.
The retriever is a registered Step (`@R.register("hybrid")`), but
it isn't pre-instantiated as a singleton component — there's no
RETRIEVER enum slot. Instead, each MCP shell builds its own
HybridRetriever the first time it's called, sharing the calling
step's `app_context` (so the lookup of `file_store`/`as_llm` works)
and forwarding any retriever knobs (`vector_weight`, `graph_*`, …)
from the step's kwargs as constructor defaults.
Callers can also pass a pre-built `BaseRetriever` instance via
`kwargs["retriever"]` — useful for tests / Python callers that want
to inject a stub.
"""
injected = step.kwargs.get("retriever")
if isinstance(injected, BaseRetriever):
return injected
cached = _RETRIEVER_CACHE.get(id(step))
if cached is not None:
return cached
backend = injected if isinstance(injected, str) else "hybrid"
cls = R.get(ComponentEnum.STEP, backend)
if cls is None or not (isinstance(cls, type) and issubclass(cls, BaseRetriever)):
# Fall back to the canonical implementation; lets configs that
# don't override `retriever` work out of the box.
cls = HybridRetriever
knob_keys = (
"vector_weight", "graph_weight", "graph_depth", "graph_decay",
"graph_direction", "graph_mode", "graph_per_path_cap",
"candidate_multiplier", "anchor_expand", "file_store",
)
init_kwargs = {k: step.kwargs[k] for k in knob_keys if k in step.kwargs}
init_kwargs["app_context"] = step.app_context
instance = cls(**init_kwargs)
_RETRIEVER_CACHE[id(step)] = instance
return instance
def _serialize_chunk(chunk, file_store, extras: dict | None = None) -> dict:
"""Serialize a FileChunk for search-result payloads.
Joins the owning file's metadata (frontmatter + st_mtime) so callers
don't have to fire a `memory_get` per result just to read `category`,
`status`, dates, etc. The join is a single in-memory dict lookup —
cost is negligible vs. the round-trip we save.
`extras` lets the caller attach step-specific fields (e.g. `graph_hop`).
"""
item = chunk.model_dump(exclude_none=True, exclude={"embedding"})
meta = file_store.get_file_meta(chunk.path)
if meta is not None:
item["file_metadata"] = meta.metadata
item["file_st_mtime"] = meta.st_mtime
else:
item["file_metadata"] = None
item["file_st_mtime"] = None
if extras:
item.update(extras)
return item
@R.register("memory_search")
class MemorySearch(BaseStep):
"""Pure-relevance retrieval (V + K hybrid). Delegates to the Retriever service."""
async def execute(self):
assert self.context is not None, "Context is not set"
query: str = self.context.get("query", "").strip()
min_score: float = self.context.get("min_score", 0.1)
max_results: int = self.context.get("max_results", 5)
assert query, "Query cannot be empty"
assert (
isinstance(min_score, float | int) and 0.0 <= min_score <= 1.0
), f"min_score must be between 0 and 1, got {min_score}"
assert (
isinstance(max_results, int) and max_results > 0
), f"max_results must be a positive integer, got {max_results}"
chunk_filter = memory_io.make_chunk_filter(
self.file_store,
paths=self.context.get("paths") or None,
tags=self.context.get("tags") or None,
exclude_paths=self.context.get("exclude_paths") or None,
)
retriever = _resolve_retriever(self)
results = await retriever.search(
query=query,
max_results=max_results,
min_score=min_score,
chunk_filter=chunk_filter,
)
payload = [_serialize_chunk(r, self.file_store) for r in results]
_set_answer(self.context, payload)
@R.register("memory_graph_search")
class MemoryGraphSearch(BaseStep):
"""V + K + graph BFS fusion. Delegates to the Retriever service.
Per-call overrides for fusion knobs (vector_weight, graph_weight,
graph_depth, graph_decay, graph_direction, graph_mode,
graph_per_path_cap, anchor_expand) are forwarded if present in the
RuntimeContext; otherwise the retriever falls back to its own
constructor defaults. These knobs are intentionally NOT exposed via
MCP — they're internal-Python-caller tuning.
"""
_OVERRIDE_KEYS = (
"vector_weight",
"graph_weight",
"graph_depth",
"graph_decay",
"graph_direction",
"graph_mode",
"graph_per_path_cap",
"anchor_expand",
)
async def execute(self):
assert self.context is not None
ctx = self.context
query: str = ctx.get("query", "").strip()
max_results: int = int(ctx.get("max_results", 5))
min_score: float = float(ctx.get("min_score", 0.0))
explicit_seeds: list[str] = list(ctx.get("seeds") or [])
assert query or explicit_seeds, "query or seeds must be provided"
assert max_results > 0
chunk_filter = memory_io.make_chunk_filter(
self.file_store,
paths=ctx.get("paths") or None,
tags=ctx.get("tags") or None,
exclude_paths=ctx.get("exclude_paths") or None,
)
# Forward per-call overrides only if the caller actually set them;
# the retriever falls back to its own defaults for missing keys.
overrides = {k: ctx.get(k) for k in self._OVERRIDE_KEYS if ctx.get(k) is not None}
retriever = _resolve_retriever(self)
results, hops = await retriever.graph_search(
query=query,
seeds=explicit_seeds,
max_results=max_results,
min_score=min_score,
chunk_filter=chunk_filter,
**overrides,
)
payload = []
for c in results:
extras = {}
hop = hops.get(c.path)
if hop is not None:
extras["graph_hop"] = hop
payload.append(_serialize_chunk(c, self.file_store, extras))
_set_answer(ctx, payload)

418
reme2/mcp/steps/sync.py Normal file
View file

@ -0,0 +1,418 @@
"""sync — continuously sync key materials into an event folder.
Layout (one folder per logical thread):
events/{date}/{name}/
├── {name}.md # index: frontmatter + narrative + Materials footer
├── {material_filename} # raw artifact 1
├── ...
└── {material_filename} # raw artifact N
Hot-path write entry. The agent picks a stable `name` per logical
thread and calls `sync` repeatedly through the task — each call
extends the same event folder rather than creating a new one. This
turns discrete writes into a coherent stream and lets PreCompact /
SessionEnd hooks treat sync as the last-chance flush before context
loss.
Behavior:
* If `events/{date}/{name}/` does NOT exist → create new folder,
write index `{name}.md` (status=active), write materials.
* If it exists with `status: active` → APPEND:
- new content → `## Update — {iso}` section appended to the body
- new materials → siblings (auto-suffix on filename collision)
- Materials footer regenerated as the trailing section, listing
every artifact actually present in the folder
- frontmatter `topics` + `tags` unioned, `updated` set to today
* If it exists with `status: distilled` / `archived` → REFUSE,
return suggested_name so the agent can start a new thread.
Zero LLM cost.
"""
import json
import os
import re
from datetime import date as date_type, datetime, timezone
from pathlib import Path
import frontmatter
from pydantic import ValidationError
from reme2.component import R
from reme2.component.base_step import BaseStep
from reme2.memory.memory_io import write_create
from reme2.schema.vault import Event
from reme2.utils.vault_paths import event_path, next_suffixed_stem
_SAFE_FILENAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
_MATERIALS_HEADER = "## Materials"
@R.register("sync")
class Sync(BaseStep):
"""Upsert into an event folder under events/{date}/{name}/.
Inputs (RuntimeContext):
name (str, required): kebab-case event identifier; becomes both
the parent dir and the index filename stem.
Reuse the same name across calls in one
thread to keep extending the same folder.
description (str): one-line summary for index frontmatter
(only set on initial create).
content (str): markdown body. On create it's the initial
body; on append it's added under a new
`## Update — {iso}` section.
topics (list[str]): related topic wikilinks; merged (union)
into frontmatter on append.
tags (list[str]): free-form tags; merged (union) on append.
materials (list[dict]): [{filename, content}, ...] — raw artifacts
written as siblings of the index. Filename
collisions auto-suffix (foo.txt → foo-2.txt).
Index body's Materials footer regenerated
each call from actual folder contents.
on_date (str | None): ISO date for the events/{date}/ bucket.
Defaults to today.
origin_session_id (str): optional source session identifier (only
set on initial create).
Output (context.response.answer):
JSON {path, materials: [paths of NEW materials this call],
created: bool, action: "created"|"appended"} on success;
{error, ...} on failure (including refusal when existing event
has status != "active").
"""
def __init__(self, vault_root: str = "", events_dir: str = "events", **kwargs):
super().__init__(**kwargs)
self.vault_root = vault_root
self.events_dir = events_dir
def _root(self) -> Path:
if self.vault_root:
return Path(self.vault_root)
watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[union-attr]
return Path(watcher.watch_path)
@staticmethod
def _validate_materials(materials: list, index_filename: str) -> tuple[list[dict], str | None]:
"""Sanity-check the materials list. Returns (cleaned, error_message)."""
cleaned: list[dict] = []
seen_in_call: set[str] = set()
for i, m in enumerate(materials):
if not isinstance(m, dict):
return [], f"materials[{i}] must be an object {{filename, content}}"
fname = m.get("filename")
if not isinstance(fname, str) or not fname:
return [], f"materials[{i}].filename is required"
if not _SAFE_FILENAME_RE.match(fname):
return [], (
f"materials[{i}].filename {fname!r} is unsafe — only "
f"letters / digits / dot / underscore / dash allowed"
)
if fname == index_filename:
return [], f"materials[{i}].filename {fname!r} collides with the index file"
if fname in seen_in_call:
return [], f"materials[{i}].filename {fname!r} duplicated within the same call"
seen_in_call.add(fname)
content = m.get("content", "")
if not isinstance(content, str):
return [], f"materials[{i}].content must be a string"
cleaned.append({"filename": fname, "content": content})
return cleaned, None
@staticmethod
def _strip_materials_footer(body: str) -> str:
"""Drop our trailing `## Materials` footer if present; return narrative."""
if not body:
return ""
# Match the footer at end-of-doc: `## Materials\n\n- [...](./...)\n` repeated.
# Cheaper rule: find the LAST `## Materials` heading at line start; strip
# from there to EOF. Whatever the user wrote above stays intact.
m = re.search(r"(?:\A|\n)##\s+Materials[ \t]*\n", body)
if m is None:
return body.rstrip()
# Find the LAST such heading by scanning all matches.
last = None
for hit in re.finditer(r"(?:\A|\n)##\s+Materials[ \t]*\n", body):
last = hit
assert last is not None
cut = last.start()
# If the heading was at offset 0 (no leading \n), keep nothing before;
# otherwise keep up to (but not including) the leading \n.
return body[:cut].rstrip()
@staticmethod
def _emit_body(narrative: str, material_filenames: list[str]) -> str:
"""Assemble body = narrative (possibly empty) + Materials footer."""
narrative = (narrative or "").rstrip()
if not material_filenames:
return f"{narrative}\n" if narrative else ""
listing = "\n".join(f"- [{f}](./{f})" for f in material_filenames)
if narrative:
return f"{narrative}\n\n{_MATERIALS_HEADER}\n\n{listing}\n"
return f"{_MATERIALS_HEADER}\n\n{listing}\n"
@staticmethod
def _resolve_filename(existing: set[str], requested: str) -> str:
"""Auto-suffix `foo.txt` → `foo-2.txt` (then -3, -4, …) on collision."""
if requested not in existing:
return requested
if "." in requested:
stem, _, ext = requested.rpartition(".")
n = 2
while f"{stem}-{n}.{ext}" in existing:
n += 1
return f"{stem}-{n}.{ext}"
n = 2
while f"{requested}-{n}" in existing:
n += 1
return f"{requested}-{n}"
@staticmethod
def _list_existing_materials(folder: Path, index_filename: str) -> list[str]:
"""Filenames in `folder` excluding the index, sorted for stability."""
if not folder.is_dir():
return []
return sorted(
entry.name for entry in folder.iterdir()
if entry.is_file() and entry.name != index_filename
)
@staticmethod
def _union(prior: list, incoming: list) -> list:
"""Order-preserving union: keep prior order, append new items in input order."""
out = list(prior)
seen = set(prior)
for item in incoming:
if item not in seen:
out.append(item)
seen.add(item)
return out
def _set_error(self, payload: dict) -> None:
assert self.context is not None
self.context.response.success = False
self.context.response.answer = json.dumps(payload, ensure_ascii=False)
def _set_ok(self, payload: dict) -> None:
assert self.context is not None
self.context.response.success = True
self.context.response.answer = json.dumps(payload, ensure_ascii=False)
async def execute(self):
assert self.context is not None
name: str = self.context.get("name", "") or ""
description: str = self.context.get("description", "") or ""
content: str = self.context.get("content", "") or ""
topics: list[str] = list(self.context.get("topics") or [])
tags: list[str] = list(self.context.get("tags") or [])
materials_in = list(self.context.get("materials") or [])
on_date = self.context.get("on_date")
origin_session_id = self.context.get("origin_session_id")
assert name, "name is required"
target = event_path(self._root(), name, on_date, self.events_dir)
materials, mat_err = self._validate_materials(materials_in, target.name)
if mat_err is not None:
self._set_error({"error": mat_err})
return
if target.exists():
await self._append(target, content, materials, topics, tags)
else:
await self._create(
target, name, description, content, materials, topics, tags,
on_date, origin_session_id,
)
async def _create(
self, target: Path, name: str, description: str, content: str,
materials: list[dict], topics: list[str], tags: list[str],
on_date, origin_session_id,
) -> None:
today = date_type.today().isoformat()
on_date_str = (
on_date.isoformat() if isinstance(on_date, date_type)
else (on_date or today)
)
metadata: dict = {
"title": name,
"description": description,
"category": "event",
"status": "active",
"tags": tags,
"topics": topics,
"created": on_date_str,
"updated": today,
}
if origin_session_id:
metadata["originSessionId"] = origin_session_id
try:
Event.model_validate(metadata)
except ValidationError as e:
self._set_error({
"error": "Event schema validation failed",
"details": e.errors(include_context=False, include_url=False),
})
return
graph = self.file_store
conflicts = graph.collisions_after_create(target)
if conflicts:
taken = {Path(p).stem for p in graph.nodes}
suggested_name = next_suffixed_stem(taken, name)
self._set_error({
"error": (
f"stem `[[{name}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this create"
),
"conflicts": conflicts,
"suggested_name": suggested_name,
"hint": (
f"retry with name='{suggested_name}', or pick a "
f"semantic qualifier (e.g. '{name}-followup')."
),
})
return
material_filenames = [m["filename"] for m in materials]
index_body = self._emit_body(content, material_filenames)
ok, payload = write_create(
self.file_store, target,
metadata=metadata, content=index_body,
)
if not ok:
self._set_error({
"path": str(target.resolve()),
"error": payload.get("error", "create failed"),
"details": payload,
})
return
material_paths: list[str] = []
for m in materials:
material_path = target.parent / m["filename"]
try:
material_path.write_text(m["content"], encoding="utf-8")
except Exception as e:
self.logger.warning(
f"sync: failed to write material {m['filename']}: {e}",
)
continue
material_paths.append(str(material_path.resolve()))
self._set_ok({
"path": str(target.resolve()),
"category": "event",
"status": "active",
"topics": topics,
"materials": material_paths,
"created": True,
"action": "created",
})
async def _append(
self, target: Path, content: str, materials: list[dict],
topics: list[str], tags: list[str],
) -> None:
# Read current frontmatter + body.
try:
raw = target.read_text(encoding="utf-8")
except Exception as e:
self._set_error({"path": str(target.resolve()), "error": f"read failed: {e}"})
return
post = frontmatter.loads(raw)
meta = dict(post.metadata)
status = meta.get("status")
if status != "active":
# Don't extend a distilled / archived thread — make the agent pick
# a new name so the prior cognition isn't silently mutated.
graph = self.file_store
taken = {Path(p).stem for p in graph.nodes}
base = target.stem
suggested = next_suffixed_stem(taken, base)
self._set_error({
"path": str(target.resolve()),
"error": (
f"event `{base}` already exists with status={status!r}; "
f"pick a new name to start a fresh thread"
),
"status": status,
"suggested_name": suggested,
})
return
folder = target.parent
existing_filenames = self._list_existing_materials(folder, target.name)
existing_set = set(existing_filenames)
# Resolve filename collisions for new materials.
resolved: list[tuple[str, str]] = [] # (filename_on_disk, content)
all_taken = set(existing_set)
for m in materials:
fname = self._resolve_filename(all_taken, m["filename"])
all_taken.add(fname)
resolved.append((fname, m["content"]))
# Write new materials to disk.
new_material_paths: list[str] = []
for fname, mcontent in resolved:
material_path = folder / fname
try:
material_path.write_text(mcontent, encoding="utf-8")
except Exception as e:
self.logger.warning(f"sync: failed to write material {fname}: {e}")
continue
new_material_paths.append(str(material_path.resolve()))
existing_filenames.append(fname)
# Rebuild the index body: narrative (existing + optional new Update
# section) + Materials footer (regenerated from disk).
narrative = self._strip_materials_footer(post.content)
if content.strip():
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
update_section = f"## Update — {ts}\n\n{content.rstrip()}\n"
narrative = (
f"{narrative.rstrip()}\n\n{update_section}"
if narrative else update_section
)
all_filenames_sorted = sorted(set(existing_filenames))
new_body = self._emit_body(narrative, all_filenames_sorted)
# Update frontmatter: union topics/tags, bump updated.
meta["topics"] = self._union(list(meta.get("topics") or []), topics)
meta["tags"] = self._union(list(meta.get("tags") or []), tags)
meta["updated"] = date_type.today().isoformat()
try:
Event.model_validate(meta)
except ValidationError as e:
self._set_error({
"path": str(target.resolve()),
"error": "Event schema validation failed on append",
"details": e.errors(include_context=False, include_url=False),
})
return
new_post = frontmatter.Post(new_body, **meta)
try:
target.write_text(frontmatter.dumps(new_post), encoding="utf-8")
except Exception as e:
self._set_error({"path": str(target.resolve()), "error": f"write failed: {e}"})
return
self._set_ok({
"path": str(target.resolve()),
"category": "event",
"status": "active",
"topics": meta["topics"],
"materials": new_material_paths,
"created": False,
"action": "appended",
})

142
reme2/mcp/steps/topic_create.py Normal file
View file

@ -0,0 +1,142 @@
"""topic_create — create a new topic under topics/{folder}/{name}.md.
The only file-creation entry point for topics. Enforces:
- path template (topics/{folder}/{name}.md)
- folder topic identification (folder == name)
- judgment-category strong-confidence (via Topic.model_validator)
- wikilink uniqueness (creating this file must not make `[[name]]` ambiguous)
Writes via `write_create` — the canonical L1 invariant gate that also
performs the wikilink-uniqueness check. The Ingestor's R-M-W loop is
reserved for content-driven flows; topic creation is path-driven, so
the direct call is sufficient.
"""
import json
from datetime import date
from pathlib import Path
from pydantic import ValidationError
from reme2.component import R
from reme2.component.base_step import BaseStep
from reme2.memory.memory_io import write_create
from reme2.schema.vault import Topic
from reme2.utils.vault_paths import next_suffixed_stem, topic_path
@R.register("topic_create")
class TopicCreate(BaseStep):
"""Create a topic file with the standard frontmatter template."""
def __init__(self, vault_root: str = "", topics_dir: str = "topics", **kwargs):
super().__init__(**kwargs)
self.vault_root = vault_root
self.topics_dir = topics_dir
def _root(self) -> Path:
if self.vault_root:
return Path(self.vault_root)
watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[union-attr]
return Path(watcher.watch_path)
async def execute(self):
assert self.context is not None
folder: str = self.context.get("folder", "")
name: str = self.context.get("name", "")
category: str = self.context.get("category", "")
description: str = self.context.get("description", "") or ""
content: str = self.context.get("content", "") or ""
confidence = self.context.get("confidence")
market = self.context.get("market")
ticker = self.context.get("ticker")
tags: list[str] = self.context.get("tags") or []
assert folder, "folder is required"
assert name, "name is required"
assert category, "category is required"
target = topic_path(self._root(), folder, name, self.topics_dir)
if target.exists():
self.context.response.success = False
self.context.response.answer = json.dumps({
"path": str(target.resolve()),
"error": "topic already exists; use memory_update / memory_property_update to modify",
}, ensure_ascii=False)
return
today = date.today().isoformat()
metadata: dict = {
"title": name,
"description": description,
"category": category,
"tags": tags,
"created": today,
"updated": today,
}
if confidence is not None:
metadata["confidence"] = confidence
if market is not None:
metadata["market"] = market
if ticker is not None:
metadata["ticker"] = ticker
# Topic schema validation (judgment categories require confidence).
try:
Topic.model_validate(metadata)
except ValidationError as e:
self.context.response.success = False
self.context.response.answer = json.dumps({
"error": "Topic schema validation failed",
"details": e.errors(include_context=False, include_url=False),
}, ensure_ascii=False)
return
# Pre-check uniqueness so we can surface a `suggested_name` to the
# agent. The actual write also routes through the same gate inside
# write_create, so this is a UX nicety, not a correctness check.
graph = self.file_store
conflicts = graph.collisions_after_create(target)
if conflicts:
taken = {Path(p).stem for p in graph.nodes}
suggested_name = next_suffixed_stem(taken, name)
self.context.response.success = False
self.context.response.answer = json.dumps({
"error": (
f"stem `[[{name}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this create"
),
"conflicts": conflicts,
"suggested_name": suggested_name,
"hint": (
f"retry with name='{suggested_name}' (numeric suffix), "
f"OR use a domain-specific qualifier (e.g. '{name}-Inc' / "
f"'{name}-v2') — semantic names beat numeric. If you "
f"actually meant the existing topic, call memory_get on "
f"one of `conflicts` instead."
),
}, ensure_ascii=False)
return
ok, payload = write_create(
self.file_store, target,
metadata=metadata, content=content,
)
if not ok:
self.context.response.success = False
self.context.response.answer = json.dumps({
"path": str(target.resolve()),
"error": payload.get("error", "create failed"),
"details": payload,
}, ensure_ascii=False)
return
is_folder_topic = folder == name
self.context.response.success = True
self.context.response.answer = json.dumps({
"path": str(target.resolve()),
"category": category,
"is_folder_topic": is_folder_topic,
"created": True,
}, ensure_ascii=False)

22
reme2/memory/__init__.py Normal file
View file

@ -0,0 +1,22 @@
"""Memory subsystem — the three services on top of the core engine.
Per the architecture blueprint:
- retriever.py Read service. V + K + Graph BFS fusion + intent
routing. Registered as a Step (`backend: hybrid`);
MCP step shells in `reme2.mcp.steps.memory_retriever`
instantiate it on demand.
- ingestor.py Cold-write service. LLM-driven R-M-W curator.
Triggered on explicit handoff (task end / SessionEnd).
- maintainer.py Treatment service. Background Merge / Split / Decay /
Lint, woken by cron or thresholds.
- summarizer.py Auxiliary used by the services.
Hot-write MCP step shells (sync, topic_create, memory_*) live in
`reme2.mcp.steps`, NOT here — they bypass services and write MFS
directly. Importing this package triggers @R.register on the three
services so configs that name them resolve at boot.
"""
from . import retriever # noqa: F401 -- runs @R.register("hybrid")
from . import maintainer # noqa: F401 -- runs @R.register("maintainer")

187
reme2/memory/ingestor.py Normal file
View file

@ -0,0 +1,187 @@
"""Smart Ingestor — ReAct agent over the Memory File System.
Per `structure.md` L31-44, the Ingestor is the SSOT engine: the **single
write entry point** to the markdown vault. Every mutation (create, body
edit, frontmatter flip, rename, delete, archive) flows through here.
Mirrors `Summarizer`'s pattern — drives a `ReActAgent` whose toolkit is
built by `memory_io.build_memory_toolkit`. The agent runs its own R-M-W
loop: read related files via tools, decide which ones to mutate, call
the right write tool. Every write tool records into an audit list, so
the caller gets a deterministic mutation trail regardless of how the
agent's reasoning unfolded.
When no LLM is configured, falls back to a direct create from
`target_path` + `metadata` + `content`. Edits/renames/deletes are not
available without an LLM.
"""
from __future__ import annotations
import datetime
import json
import zoneinfo
from pathlib import Path
from agentscope.agent import ReActAgent
from agentscope.message import Msg
from agentscope.tool import Toolkit
from pydantic import BaseModel, Field
from ..component.runtime_response import _set_answer, _to_jsonable
from . import memory_io
from .memory_io import MemoryIO, write_create
from ..component import R
from ..component.base_step import BaseStep
from ..enumeration import ComponentEnum
from ..utils.wikilink import extract_wikilinks
class IngestResult(BaseModel):
"""Audit trail for a single ingest call."""
applied: list[dict] = Field(default_factory=list, description="Successful ops with paths.")
rejected: list[dict] = Field(default_factory=list, description="Ops the validator refused.")
failed: list[dict] = Field(default_factory=list, description="Ops that errored at apply time.")
skipped: bool = Field(default=False, description="True if the LLM returned a SkipOp.")
used_llm: bool = Field(default=False, description="False = degraded path (no LLM configured).")
@property
def success(self) -> bool:
"""Skipped or any-applied with no failures = success."""
if self.skipped:
return True
return len(self.applied) > 0 and len(self.failed) == 0
@R.register("ingestor")
class Ingestor(BaseStep):
"""R-M-W ingestor exposed as a single-write-entry MCP tool.
Inputs (read from RuntimeContext):
content (str, required): the material being ingested.
hint (str, optional): caller guidance to the LLM.
target_path (str, optional): suggested file path; required for
the degraded path (no LLM).
metadata (dict, optional): suggested frontmatter; used by
the degraded path or as a hint to the LLM.
related_paths (list[str], optional): explicit related files,
auto-extended with wikilinks parsed from `content`.
Output (written to context.response.answer):
IngestResult JSON — applied/failed lists plus used_llm flag and
the agent's final-message summary.
"""
def __init__(
self,
toolkit: Toolkit | None = None,
console_enabled: bool = False,
timezone: str | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.toolkit = toolkit
self.console_enabled = console_enabled
self.timezone = timezone
def _now(self) -> datetime.datetime:
if self.timezone:
try:
return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone))
except Exception as e:
self.logger.error(f"Invalid timezone: {self.timezone}, error={e}")
return datetime.datetime.now()
def _vault_root(self) -> Path:
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
if watcher is None:
vr = getattr(self.file_store, "vault_root", None)
return Path(vr).resolve() if vr else Path.cwd().resolve()
return Path(getattr(watcher, "watch_path", ".")).resolve()
async def execute(self):
assert self.context is not None
content: str = self.context.get("content", "") or ""
hint: str = self.context.get("hint", "") or ""
target_path: str = self.context.get("target_path") or ""
metadata: dict = dict(self.context.get("metadata") or {})
related_paths: list[str] = list(self.context.get("related_paths") or [])
assert content, "content is required"
# Auto-discover wikilink targets in content as a hint for the agent.
for link in extract_wikilinks(content):
hit = memory_io.wikilink_lookup(self.file_store, link)["path"]
if hit and hit not in related_paths:
related_paths.append(hit)
as_llm = self._get_component_optional(ComponentEnum.AS_LLM, "default", "model")
if as_llm is None:
result = self._degraded(target_path, metadata, content)
self.context.response.success = result.success
_set_answer(self.context, result.model_dump())
return
vault_root = self._vault_root()
mio = MemoryIO(self.file_store, vault_root)
toolkit = mio.register_all(self.toolkit)
agent = ReActAgent(
name="reme_ingestor",
model=self.as_llm,
sys_prompt=self.prompt_format("system_prompt", vault_root=str(vault_root)),
formatter=self.as_llm_formatter,
toolkit=toolkit,
)
agent.set_console_output_enabled(self.console_enabled)
user_message: str = self.prompt_format(
"user_message",
today=self._now().strftime("%Y-%m-%d"),
vault_root=str(vault_root),
hint=hint or "(none)",
target_path=target_path or "(none)",
metadata=json.dumps(_to_jsonable(metadata), ensure_ascii=False),
related=json.dumps(related_paths, ensure_ascii=False),
content=content,
)
final_msg: Msg = await agent.reply(
Msg(name="reme", role="user", content=user_message),
)
summary = final_msg.get_text_content() or ""
result = IngestResult(used_llm=True)
for entry in mio.audit:
(result.applied if entry.get("ok") else result.failed).append(entry)
if not mio.audit and summary.strip().upper().startswith("SKIP"):
result.skipped = True
self.context.response.success = result.success
payload = result.model_dump()
payload["agent_summary"] = summary
_set_answer(self.context, payload)
def _degraded(self, target_path: str, metadata: dict, content: str) -> IngestResult:
"""Without an LLM, only direct create from explicit target_path is
supported. Useful for tests and bootstrap scripts."""
result = IngestResult(used_llm=False)
if not target_path:
result.failed.append({
"op": "create",
"ok": False,
"error": "degraded path: target_path is required when no LLM is configured",
})
return result
path = Path(target_path)
if not path.is_absolute():
path = self._vault_root() / path
ok, payload = write_create(
self.file_store, path,
metadata=metadata, content=content,
)
bucket = result.applied if ok else result.failed
bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload})
return result

110
reme2/memory/ingestor.yaml Normal file
View file

@ -0,0 +1,110 @@
system_prompt: |
You are the memory curator for a markdown vault — the LLM-driven
R-M-W loop over EXISTING files. You only run when the agent
EXPLICITLY HANDS OFF: at task completion, at session end, or when
the agent decides the working set is ready to be distilled. By that
point the agent has typically called `sync` along the way to
land raw facts as event folders. Your job: take whatever materials
the agent feeds you (inline text and/or paths), read them + any
related topics, decide which existing topics need updates, which
deserve a new topic, then flip each distilled event's status.
vault_root: {vault_root}
# Scope of THIS tool
- You are the COLD path. You DO NOT run on every turn — only on
explicit handoff (task done / session ending / agent invokes you
directly). The hot path (`sync`, deterministic, no LLM)
has already preserved raw facts continuously through the task.
- The agent supplies the working set in two interchangeable forms:
* `content` — inline material the agent is giving you directly
(a distillation hint, a session summary, raw text it wants
folded into the graph).
* `related_paths` — paths the agent points you at (event folder
indexes, individual material files, candidate topics it flagged
for update). Both forms can appear together; treat them as a
single working set.
- An event is a FOLDER containing the index `{name}.md` plus
materials (raw conversation snippets, tool outputs, data dumps).
Whenever a path in `related_paths` is an event index, `memory_get`
the index first, then `memory_get` any materials whose content you
need (the index lists them under `## Materials`).
# Vault conventions
- Topics live under `topics/{{folder}}/{{name}}.md`. A folder topic
has folder == name.
- Events live under `events/{{YYYY-MM-DD}}/{{name}}/{{name}}.md` —
this is an INDEX inside a folder; sibling files are materials.
- Frontmatter is YAML; `category`, `created`, `updated`, `tags`,
`status` are common.
- Cross-file references use `[[wikilink]]` syntax (stem-form `[[X]]`
or path-form `[[topics/X/X]]`).
# Available tools
Read tools (use these to gather context BEFORE writing):
- memory_get(path, include_chunks=False): full file content + frontmatter.
Call this on each event index AND on the materials it lists.
- memory_list(path_prefix=None, tags=None, metadata=None, limit=100):
list indexed files filtered by prefix / tags / frontmatter.
- memory_resolve_wikilink(wikilink): resolve `[[X]]` to a path.
- memory_backlinks(path): files linking to a given path.
- memory_links(path): files a given path links to.
Write tools (each returns success + payload; mutations are SSOT-routed):
- memory_update(path, old_string, new_string, replace_all=False):
body edit by exact-string substitution. Use a tail snippet to append.
- memory_property_update(path, key, value): change one frontmatter
key (value=null deletes it). After distilling an event into one or
more topics, flip that event's status to "distilled" with this.
- memory_create(path, metadata, content, overwrite=False, force=False):
new file. Reserve for genuinely NEW topics — do NOT use this to log
events; `sync` (deterministic) owns events. ALL paths must
be ABSOLUTE under vault_root.
- memory_rename(old_path, new_path): move file + rewrite cross-vault
wikilinks.
- memory_delete(path): remove a file.
- memory_archive(path): flip `status: archived` and move under
`<vault>/Archive/`.
# Decision rules
1. If material is ALREADY covered by existing topics → reply with a
single line `SKIP: <one-line reason>` and call no tools.
2. If material CONTRADICTS an existing block → memory_update with a
unique snippet of the outdated text and the corrected replacement.
3. If material EXTENDS an existing topic → memory_update using a
unique TAIL snippet of the existing body, with new_string =
tail + blank line + new content.
4. If material warrants a GENUINELY NEW topic → memory_create at
`topics/{{folder}}/{{name}}.md`. Do NOT memory_create under events/.
5. After integrating an event's content into a topic, flip that event's
status to "distilled" with memory_property_update.
6. Never delete unless the material explicitly asks for deletion.
7. Keep edits minimal — read related files first, edit only what
must change.
8. Always include reasonable frontmatter on memory_create — at minimum
`title`, `category`, `created`, `updated`. Use today's date for
`created` / `updated`.
9. After all writes, end with a one-paragraph summary of what you did
and why.
user_message: |
# CONTEXT
today: {today}
vault_root: {vault_root}
# WORKING SET (handed off by the agent)
caller hint: {hint}
target_path hint: {target_path}
metadata hint: {metadata}
related paths (auto-discovered from wikilinks in content + caller's related_paths): {related}
content (inline material the agent is feeding you — distillation
hint, session summary, or raw text):
{content}
Treat `content` (inline) and `related paths` (pointers) as a single
working set. For each related path that's an event index, read the
index then read any materials it lists. Inspect the related topics,
then perform the minimal set of writes needed to distill this
material into the topic graph. Flip any distilled event's status as
part of the same call. End with a one-paragraph summary.

511
reme2/memory/maintainer.py Normal file
View file

@ -0,0 +1,511 @@
"""Maintainer — treatment service. One pass, four signals, one plan.
Per the architecture blueprint, the Maintainer is the third memory
service alongside the Retriever (read) and the Ingestor (cold-write).
It's woken by cron or thresholds, not by per-turn agent calls.
DESIGN: single `Maintainer` Step, NOT four. Vault hygiene is one
operation with four signal sources that contend for the same files —
splitting them into independent Steps would let merge & split reverse
each other, let decay archive a file that merge wanted to absorb, and
force every cron tick to scan the vault four times. So the Maintainer
follows a plan-then-apply pipeline:
scan_signals() # one walk, all signals shared
↓
propose_*() # each signal source emits Op records
↓
resolve_conflicts() # data-driven matrix dedupes / orders ops
↓
apply() # lint → decay → merge → split, dry-run aware
↓
audit # unified trail returned to caller
OPERATIONS
LintFinding read-only diagnostic; never conflicts.
DecayOp move a stale event under <vault>/<archive_dir>/.
MergeOp absorb sources[] into canonical; rewrite incoming
wikilinks; archive sources.
SplitOp extract sections of source[] into new files; replace
in original with [[…]] stubs.
CONFLICT MATRIX (resolved before apply)
MergeOp(source=P) ⊕ SplitOp(source=P) → drop split
MergeOp(source=P) ⊕ DecayOp(path=P) → drop decay
SplitOp(source=P) ⊕ DecayOp(path=P) → drop split
MergeOp(canonical=A,…) × N (same A) → union sources
SplitOp(source=P) × N → keep highest confidence
LintFinding ⊕ anything → coexist
APPLY ORDER is fixed: lint → decay → merge → split. This guarantees
that by the time split runs, merge has already rewritten paths; if a
split target was absorbed by a merge, apply skips it and records a
`stale_target` audit entry instead of crashing.
LLM-DRIVEN PROPOSERS (merge/split) are scaffolded — the structure +
op shape + conflict path are wired and verified, but the LLM-backed
similarity / split heuristics are pending design and currently return
empty proposal lists. Lint and decay are fully implemented.
"""
from __future__ import annotations
import datetime
from collections import defaultdict
from pathlib import Path
from typing import Literal
from pydantic import BaseModel, Field
from ..component import R
from ..component.base_step import BaseStep
from ..component.runtime_response import _set_answer
from ..enumeration import ComponentEnum
from ..schema.vault.registry import schema_for
from . import memory_io
# ---------------------------------------------------------------------------
# Op records
# ---------------------------------------------------------------------------
class LintFinding(BaseModel):
"""Read-only diagnostic. Never conflicts; never mutates."""
op: Literal["lint"] = "lint"
path: str
kind: Literal["broken_wikilink", "schema_violation", "stem_collision"]
detail: str
class DecayOp(BaseModel):
"""Archive a stale event. Moves the file; doesn't change body."""
op: Literal["decay"] = "decay"
path: str
age_days: int
reason: str = "past freshness window"
class MergeOp(BaseModel):
"""Consolidate `sources` into `canonical`. Sources get archived."""
op: Literal["merge"] = "merge"
canonical: str
sources: list[str] = Field(default_factory=list)
confidence: float = 0.0
reason: str = ""
class SplitOp(BaseModel):
"""Extract sections of `source` into new sibling files."""
op: Literal["split"] = "split"
source: str
sections: list[dict] = Field(default_factory=list) # [{title, body, target_path}]
confidence: float = 0.0
reason: str = ""
# ---------------------------------------------------------------------------
# Signals: one scan, all observers consume the same dict
# ---------------------------------------------------------------------------
class FileSignal(BaseModel):
"""Per-file derived signals shared by every proposer.
Built once by `_scan_signals`. Holds only what's cheap to compute
from `file_store.nodes` + edge index — no body reads, no LLM calls.
Heavier signals (token counts, embeddings) are pulled lazily inside
the proposers that actually need them.
"""
path: str
relpath: str = "" # path relative to vault_root, "" if outside
category: str = ""
status: str = ""
age_days: int = 0
metadata: dict = Field(default_factory=dict)
declared_topics: list[str] = Field(default_factory=list) # raw [[…]] strings
# ---------------------------------------------------------------------------
# Maintainer
# ---------------------------------------------------------------------------
@R.register("maintainer")
class Maintainer(BaseStep):
"""Unified vault hygiene. Scan → propose → resolve → apply.
Reads from RuntimeContext (all optional):
ops (list[str], default ["lint","decay"]):
subset of {"lint","decay","merge","split"} to run.
Merge/split require an LLM and are off by default.
dry_run (bool, default True): if True, returns the plan
but doesn't mutate the vault.
decay_days (int, default constructor `decay_days`): freshness
window for the decay proposer.
target_prefix (str, default ""): restrict scan to relpaths
starting with this prefix (e.g. "events/").
token_threshold (int, default constructor): split threshold.
merge_threshold (float, default constructor): cluster cutoff.
Writes to ctx.response.answer:
{
"ops_run": [...], # which proposers ran
"scanned": int,
"proposed": [op,...], # raw, before conflict resolution
"plan": [op,...], # after conflict resolution
"applied": [op,...], # successfully mutated
"skipped": [op,...], # dropped by conflict resolution
"failed": [op,...], # apply errored
"dry_run": bool,
"ran_at": iso,
}
"""
# Fixed apply order — see module docstring CONFLICT MATRIX section.
_APPLY_ORDER = ("lint", "decay", "merge", "split")
def __init__(
self,
decay_days: int = 90,
archive_dir: str = "Archive",
target_status: str = "distilled",
token_threshold: int = 4000,
merge_threshold: float = 0.85,
**kwargs,
):
super().__init__(**kwargs)
self.decay_days = decay_days
self.archive_dir = archive_dir
self.target_status = target_status
self.token_threshold = token_threshold
self.merge_threshold = merge_threshold
# -- entry point --------------------------------------------------------
async def execute(self):
assert self.context is not None
params = self._load_params()
signals = self._scan_signals(target_prefix=params["target_prefix"])
proposed: list[BaseModel] = []
if "lint" in params["ops"]:
proposed.extend(self._propose_lint(signals))
if "decay" in params["ops"]:
proposed.extend(self._propose_decay(signals, params["decay_days"]))
if "merge" in params["ops"]:
proposed.extend(await self._propose_merge(signals, params["merge_threshold"]))
if "split" in params["ops"]:
proposed.extend(await self._propose_split(signals, params["token_threshold"]))
plan, dropped = self._resolve_conflicts(proposed)
applied: list[dict] = []
failed: list[dict] = []
if not params["dry_run"]:
applied, failed = await self._apply(plan)
audit = {
"ops_run": params["ops"],
"scanned": len(signals),
"proposed": [_dump(o) for o in proposed],
"plan": [_dump(o) for o in plan],
"applied": applied,
"skipped": [_dump(o) for o in dropped],
"failed": failed,
"dry_run": params["dry_run"],
"ran_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
}
self.context.response.success = len(failed) == 0
_set_answer(self.context, audit)
# -- params -------------------------------------------------------------
def _load_params(self) -> dict:
"""Pull RuntimeContext kwargs with proper defaults.
Centralizing this kills the `int(ctx.get(...))` / `float(ctx.get(...))`
pattern that bites when `.get` returns None — every conversion
below has a guaranteed-non-None operand.
"""
ctx = self.context
assert ctx is not None
ops = ctx.get("ops") or ["lint", "decay"]
if not isinstance(ops, list):
raise ValueError(f"ops must be a list of strings, got {type(ops).__name__}")
bad = [o for o in ops if o not in self._APPLY_ORDER]
if bad:
raise ValueError(f"unknown ops: {bad}; allowed: {list(self._APPLY_ORDER)}")
return {
"ops": ops,
"dry_run": bool(ctx.get("dry_run", True)),
"decay_days": int(ctx.get("decay_days") or self.decay_days),
"target_prefix": str(ctx.get("target_prefix") or ""),
"token_threshold": int(ctx.get("token_threshold") or self.token_threshold),
"merge_threshold": float(ctx.get("merge_threshold") or self.merge_threshold),
}
# -- scan ---------------------------------------------------------------
def _vault_root(self) -> Path | None:
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
if watcher is None:
return None
return Path(getattr(watcher, "watch_path", ".")).resolve()
def _scan_signals(self, *, target_prefix: str = "") -> list[FileSignal]:
"""One walk over the indexed files. Cheap signals only."""
vault_root = self._vault_root()
now = datetime.datetime.now().timestamp()
signals: list[FileSignal] = []
for path, meta in memory_io.iter_files(self.file_store):
relpath = ""
if vault_root is not None:
try:
relpath = str(Path(path).resolve().relative_to(vault_root))
except ValueError:
pass
if target_prefix and not relpath.startswith(target_prefix):
continue
fm = meta.metadata or {}
age_seconds = max(0.0, now - (meta.st_mtime or now))
signals.append(FileSignal(
path=path,
relpath=relpath,
category=str(fm.get("category") or ""),
status=str(fm.get("status") or ""),
age_days=int(age_seconds // 86400),
metadata=fm,
declared_topics=list(fm.get("topics") or []),
))
return signals
# -- proposers ----------------------------------------------------------
def _propose_lint(self, signals: list[FileSignal]) -> list[LintFinding]:
"""Broken wikilinks + frontmatter schema violations + stem collisions."""
out: list[LintFinding] = []
for sig in signals:
for link in sig.declared_topics:
if not memory_io.wikilink_lookup(self.file_store, link)["exists"]:
out.append(LintFinding(
path=sig.path, kind="broken_wikilink",
detail=f"unresolved wikilink {link!r}",
))
cls = schema_for(sig.category)
if cls is not None:
try:
cls(**sig.metadata)
except Exception as e:
out.append(LintFinding(
path=sig.path, kind="schema_violation",
detail=f"{cls.__name__}: {type(e).__name__}: {e}"[:240],
))
# Stem collisions: the engine API exposes the ambiguous-stem map.
ambig = memory_io.all_ambiguous_wikilinks(self.file_store)
for stem, paths in ambig.items():
for p in paths:
out.append(LintFinding(
path=p, kind="stem_collision",
detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}",
))
return out
def _propose_decay(
self, signals: list[FileSignal], decay_days: int,
) -> list[DecayOp]:
"""Distilled events past the freshness window."""
out: list[DecayOp] = []
for sig in signals:
if sig.category != "event":
continue
if sig.status != self.target_status:
continue
if sig.age_days < decay_days:
continue
out.append(DecayOp(
path=sig.path, age_days=sig.age_days,
reason=f"event {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
))
return out
async def _propose_merge(
self, signals: list[FileSignal], threshold: float,
) -> list[MergeOp]:
"""Cluster near-duplicate topics → MergeOps (LLM-assisted).
SCAFFOLD: the structure is wired but the clustering implementation
is pending. Returns []. When implemented:
1. Embed titles + descriptions; cluster on cosine ≥ threshold.
2. For each cluster, ask the LLM to pick the canonical and
summarize what the merged body should preserve.
3. Emit MergeOp(canonical, sources=[non-canonical], confidence,
reason=LLM justification).
"""
return []
async def _propose_split(
self, signals: list[FileSignal], token_threshold: int,
) -> list[SplitOp]:
"""Topics over the token threshold → SplitOps (LLM-assisted).
SCAFFOLD: returns []. When implemented:
1. Filter signals to topics whose body exceeds token_threshold.
2. Read body, ask the LLM for a section partition + filenames.
3. Emit SplitOp(source, sections=[{title,body,target_path}],
confidence, reason).
"""
return []
# -- conflict resolution ------------------------------------------------
def _resolve_conflicts(
self, proposed: list[BaseModel],
) -> tuple[list[BaseModel], list[BaseModel]]:
"""Apply the conflict matrix; return (kept_plan, dropped).
Steps:
1. Union MergeOps that share a canonical.
2. Dedupe SplitOps by source (highest confidence wins).
3. For each path, resolve {merge-source, split-source, decay}
contention per the matrix in the module docstring.
4. Lint findings are pass-through.
"""
lints = [o for o in proposed if isinstance(o, LintFinding)]
merges = [o for o in proposed if isinstance(o, MergeOp)]
splits = [o for o in proposed if isinstance(o, SplitOp)]
decays = [o for o in proposed if isinstance(o, DecayOp)]
dropped: list[BaseModel] = []
# (1) Union merges by canonical.
merged_by_canon: dict[str, MergeOp] = {}
for m in merges:
existing = merged_by_canon.get(m.canonical)
if existing is None:
merged_by_canon[m.canonical] = m
else:
merged_sources = list(dict.fromkeys(existing.sources + m.sources))
existing.sources = merged_sources
existing.confidence = max(existing.confidence, m.confidence)
existing.reason = (existing.reason + "; " + m.reason).strip("; ")
merges = list(merged_by_canon.values())
# (2) Dedupe splits by source.
split_by_source: dict[str, SplitOp] = {}
for s in splits:
current = split_by_source.get(s.source)
if current is None or s.confidence > current.confidence:
if current is not None:
dropped.append(current)
split_by_source[s.source] = s
else:
dropped.append(s)
splits = list(split_by_source.values())
# (3) Build path → owning op index.
merge_paths: set[str] = set()
for m in merges:
merge_paths.update(m.sources)
merge_paths.add(m.canonical)
kept_splits: list[SplitOp] = []
for s in splits:
if s.source in merge_paths:
dropped.append(s) # MergeOp ⊕ SplitOp(source=P) → drop split
else:
kept_splits.append(s)
split_sources = {s.source for s in kept_splits}
kept_decays: list[DecayOp] = []
for d in decays:
if d.path in merge_paths:
dropped.append(d) # MergeOp ⊕ DecayOp(P) → drop decay
elif d.path in split_sources:
# SplitOp(source=P) ⊕ DecayOp(P) → matrix says drop split,
# but split was kept above (no merge contention) so the
# decay wins here: archive trumps refining what's about
# to leave the active set.
# → drop the split, keep decay.
for s in list(kept_splits):
if s.source == d.path:
kept_splits.remove(s)
dropped.append(s)
kept_decays.append(d)
else:
kept_decays.append(d)
plan: list[BaseModel] = []
plan.extend(lints)
plan.extend(kept_decays)
plan.extend(merges)
plan.extend(kept_splits)
return plan, dropped
# -- apply --------------------------------------------------------------
async def _apply(
self, plan: list[BaseModel],
) -> tuple[list[dict], list[dict]]:
"""Execute the plan in fixed order. Each phase yields audit dicts.
Lint never mutates → recorded as applied no-op so the audit shows
which findings the cron run surfaced.
Decay / Merge / Split call into the existing memory_io primitives
once the heuristics land; for now they record `pending_apply` so
the dry_run=False path still produces a stable audit shape.
"""
applied: list[dict] = []
failed: list[dict] = []
# Group ops by type, apply in fixed order.
by_type: dict[str, list] = defaultdict(list)
for o in plan:
by_type[o.op].append(o) # type: ignore[attr-defined]
for kind in self._APPLY_ORDER:
for op in by_type.get(kind, []):
try:
result = await self._apply_one(op)
applied.append({"op": op.op, **result}) # type: ignore[attr-defined]
except Exception as e:
failed.append({
"op": op.op, # type: ignore[attr-defined]
"payload": _dump(op),
"error": f"{type(e).__name__}: {e}",
})
return applied, failed
async def _apply_one(self, op: BaseModel) -> dict:
if isinstance(op, LintFinding):
# Diagnostic-only: surfacing the finding IS the work.
return {"path": op.path, "kind": op.kind, "noop": True}
if isinstance(op, DecayOp):
# TODO: wire to MemoryArchive once we're ready to actually
# move files from a cron context. For now, surface intent.
return {"path": op.path, "status": "pending_apply",
"would": "flip status=archived + move to archive_dir"}
if isinstance(op, MergeOp):
return {"canonical": op.canonical, "sources": op.sources,
"status": "pending_apply",
"would": "merge bodies + rewrite incoming wikilinks + archive sources"}
if isinstance(op, SplitOp):
return {"source": op.source, "sections": len(op.sections),
"status": "pending_apply",
"would": "extract sections + replace with [[…]] stubs"}
raise TypeError(f"unknown op type: {type(op).__name__}")
def _dump(op: BaseModel) -> dict:
"""JSON-friendly snapshot of an op record."""
return op.model_dump()

849
reme2/memory/memory_io.py Normal file
View file

@ -0,0 +1,849 @@
"""Memory File System engine API — the core engine's outward surface.
The .md files are the SSOT (per `structure.md` §"核心引擎"). The engine
is layered:
Memory File System → Watcher & Parser → Projections (vector / FTS / graph)
(write entry) (incremental) (read entry, derived)
This module is the **single public API surface** over that engine. Every
consumer — MCP step shells, the three memory services (Retriever,
Ingestor, Maintainer), and the agent toolkit — talks to the engine
through these functions, not by reaching into `BaseFileStore` directly.
That keeps `file_store` an implementation detail (could be local sqlite,
remote, etc.) and gives the layering one place to evolve.
Four sections:
1. CRUD writes — write_create / delete / update / property_update
/ rename / archive. The MFS write entry.
2. MFS reads — read_file / list_files / links_of / backlinks_of
/ wikilink_lookup / count_tokens / iter_files.
Primary-key lookups against the file_store cache.
3. Projections — vector_search / keyword_search / expand_neighbors
/ extract_anchors / chunks_by_paths / make_chunk_filter
/ all_ambiguous_wikilinks. The read entry to derived
indexes (composed by Retriever into V+K+graph fusion).
4. Toolkit — `MemoryIO` class wrapping the read/write helpers as
`agentscope.tool` callables for ReActAgent (Ingestor).
Lives in `reme2/memory/` (not `reme2/mcp/`) so memory services and the
MCP transport layer can both consume it without forming an import cycle
through the transport layer.
"""
from __future__ import annotations
import json
import re
import shutil
from collections.abc import Iterable, Iterator
from pathlib import Path
from typing import Any
import frontmatter
from agentscope.message import TextBlock
from agentscope.tool import Toolkit, ToolResponse
from ..component.runtime_response import _to_jsonable
from ..schema import ChunkFilter, FileChunk, FileMetadata
from ..utils.wikilink import WIKILINK_RE
# ===========================================================================
# Section 1 — CRUD writes
# ===========================================================================
#
# Every mutation in the system funnels through these. Hot-write MCP shells
# (sync, topic_create, memory_*) call them directly; cold-write services
# (Ingestor R-M-W, Maintainer decay) compose them.
def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
"""Rewrite wikilink targets in raw text.
Only the `target` portion of `[[target]]` / `[[target#anchor]]` /
`[[target|alias]]` / `![[target]]` is replaced; anchors, aliases,
and embed prefixes are preserved.
"""
if not mapping:
return text
def sub(m: re.Match) -> str:
target_raw = m.group(1)
target = target_raw.strip()
if target in mapping:
return m.group(0).replace(target_raw, mapping[target], 1)
return m.group(0)
return WIKILINK_RE.sub(sub, text)
def write_create(
file_store,
path: Path,
metadata: dict,
content: str,
overwrite: bool = False,
force: bool = False,
) -> tuple[bool, dict]:
"""Single L1 entry point for creating a markdown file.
All file creation in the project must funnel through this so the
wikilink uniqueness invariant is enforced in exactly one place.
Refuses (returns (False, payload)) when:
- file already exists (unless overwrite=True)
- creating it would make `[[stem]]` resolve ambiguously against
the current file_store (unless force=True)
"""
if path.exists() and not overwrite:
return False, {"path": str(path), "error": "file already exists"}
if not force:
conflicts = file_store.collisions_after_create(path)
if conflicts:
return False, {
"path": str(path),
"error": (
f"stem `[[{path.stem}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this create"
),
"conflicts": conflicts,
"hint": (
f"either rename to a unique stem, or have callers "
f"link via the explicit-path form "
f"`[[{path.parent.name}/{path.stem}]]`; pass "
f"force=true only if you accept the ambiguity"
),
}
path.parent.mkdir(parents=True, exist_ok=True)
post = frontmatter.Post(content, **metadata)
path.write_text(frontmatter.dumps(post), encoding="utf-8")
return True, {"path": str(path), "created": True}
def write_delete(path: Path | str) -> tuple[bool, dict]:
"""Delete a file. Watcher removes from store + graph."""
target = Path(path)
if not target.exists():
return False, {"path": str(target), "error": "not found"}
target.unlink()
return True, {"path": str(target), "deleted": True}
def write_update(
path: Path | str,
old_string: str,
new_string: str,
replace_all: bool = False,
) -> tuple[bool, dict]:
"""Edit-style content update — replace `old_string` with `new_string`."""
target = Path(path)
if not target.is_file():
return False, {"path": str(target), "error": "file not found"}
if not old_string:
return False, {
"path": str(target),
"error": "old_string is required (use write_create to write a new file)",
}
raw = target.read_text(encoding="utf-8")
occurrences = raw.count(old_string)
if occurrences == 0:
return False, {"path": str(target), "error": "old_string not found in file"}
if occurrences > 1 and not replace_all:
return False, {
"path": str(target),
"error": f"old_string appears {occurrences} times; pass replace_all=true to replace all",
"occurrences": occurrences,
}
if replace_all:
new_raw = raw.replace(old_string, new_string)
else:
new_raw = raw.replace(old_string, new_string, 1)
target.write_text(new_raw, encoding="utf-8")
return True, {
"path": str(target),
"replaced": occurrences if replace_all else 1,
}
def write_property_update(path: Path | str, key: str, value) -> tuple[bool, dict]:
"""Update a single YAML frontmatter key. value=None deletes the key."""
target = Path(path)
if not target.is_file():
return False, {"path": str(target), "error": "file not found"}
raw = target.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
if value is None:
post.metadata.pop(key, None)
else:
post.metadata[key] = value
target.write_text(frontmatter.dumps(post), encoding="utf-8")
return True, {"path": str(target), "key": key, "value": value}
def write_rename(
file_store,
vault_root: Path | str,
old_path: Path | str,
new_path: Path | str,
) -> tuple[bool, dict]:
"""Rename a file and rewrite incoming wikilinks across the vault.
Atomically moves `old_path` to `new_path`, then rewrites short-form
`[[old_stem]]` and path-form `[[old_relative]]` wikilinks in every
file that already had a *resolved* link to old_path. Look-up uses
`file_store.get_backlinks(old_path)` so the work is O(K) where K is
the number of incoming references — not a full vault scan.
Refuses if:
- `old_path` doesn't exist
- `new_path` already exists
- the rename would make `[[new_stem]]` resolve ambiguously
"""
old_p = Path(old_path).resolve()
new_p = Path(new_path).resolve()
if not old_p.is_file():
return False, {"old_path": str(old_p), "error": "old_path not found"}
if new_p.exists():
return False, {"new_path": str(new_p), "error": "new_path already exists"}
if old_p == new_p:
return False, {"error": "old_path and new_path are the same"}
conflicts = file_store.collisions_after_create(new_p)
if conflicts:
return False, {
"error": (
f"stem `[[{new_p.stem}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this rename"
),
"conflicts": conflicts,
"hint": (
f"either rename to a unique stem (consider a "
f"domain-specific suffix), or have callers link via "
f"the explicit-path form `[[{new_p.parent.name}/{new_p.stem}]]`"
),
}
vault_root_p = Path(vault_root).resolve()
old_stem = old_p.stem
new_stem = new_p.stem
replacements: dict[str, str] = {}
if old_stem != new_stem:
replacements[old_stem] = new_stem
try:
old_rel = str(old_p.relative_to(vault_root_p).with_suffix(""))
new_rel = str(new_p.relative_to(vault_root_p).with_suffix(""))
if old_rel != new_rel:
replacements[old_rel] = new_rel
replacements[old_rel + ".md"] = new_rel + ".md"
except ValueError:
pass # paths outside vault root — skip path-form rewrite
referring_paths = [m.path for m, _ in file_store.get_backlinks(str(old_p))]
new_p.parent.mkdir(parents=True, exist_ok=True)
old_p.rename(new_p)
updated_files: list[str] = []
write_errors: list[dict] = []
if replacements and referring_paths:
for path in referring_paths:
file_path = Path(path)
if not file_path.is_file():
continue
try:
raw = file_path.read_text(encoding="utf-8")
new_raw = _replace_wikilink_targets(raw, replacements)
if new_raw != raw:
file_path.write_text(new_raw, encoding="utf-8")
updated_files.append(path)
except Exception as exc:
write_errors.append({"path": path, "error": str(exc)})
return True, {
"old_path": str(old_p),
"new_path": str(new_p),
"stem_changed": old_stem != new_stem,
"replacements": replacements,
"referring_count": len(referring_paths),
"updated_files": updated_files,
"write_errors": write_errors,
}
def write_archive(
vault_root: Path | str,
path: Path | str,
archive_dir_name: str = "Archive",
) -> tuple[bool, dict]:
"""Archive a file: flip `status: archived`, then move under `<vault>/<archive_dir>/`.
Composed by the Maintainer's decay pass when an event falls past its
freshness window. Backlinks are *not* rewritten — dangling links to
archived files are the intended audit trail.
"""
src = Path(path).resolve()
if not src.is_file():
return False, {"path": str(src), "error": "file not found"}
vault = Path(vault_root).resolve()
try:
rel = src.relative_to(vault)
except ValueError:
return False, {
"path": str(src),
"error": f"path is outside vault_root {vault}",
}
dst = vault / archive_dir_name / rel
if dst.exists():
return False, {
"path": str(src),
"error": f"archive destination already exists: {dst}",
}
ok, prop_payload = write_property_update(src, "status", "archived")
if not ok:
return False, {**prop_payload, "stage": "property_update"}
dst.parent.mkdir(parents=True, exist_ok=True)
try:
shutil.move(str(src), str(dst))
except OSError as exc:
return False, {
"path": str(src),
"error": f"move failed: {exc}",
"stage": "move",
}
return True, {
"old_path": str(src),
"new_path": str(dst),
"archived": True,
}
# ===========================================================================
# Section 2 — MFS reads (primary-key lookups against the file_store cache)
# ===========================================================================
#
# These don't touch the projection indexes — they hit the in-memory file
# meta + edge cache that the file_store maintains as a mirror of disk.
# For body / chunk content reads, `read_file` does fall through to the
# disk to get the latest text (file_store cache may lag a write).
async def read_file(
file_store,
path: str,
*,
include_chunks: bool = False,
) -> dict:
"""Read frontmatter + body for one path. Optionally include parsed chunks.
On-disk frontmatter is the source of truth — the file_store cache may
lag a write that hasn't been picked up by the watcher yet.
"""
meta = file_store.get_file_meta(path)
result: dict = {"path": path, "exists": False}
if meta is not None:
edges = file_store.get_edges(path)
result.update({
"exists": True,
"metadata": meta.metadata,
"link": [e.model_dump(exclude_none=True) for e in edges],
})
file_path = Path(path)
if file_path.is_file():
raw = file_path.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
result["exists"] = True
result["content"] = post.content
result["metadata"] = dict(post.metadata)
if include_chunks:
chunks = await file_store.get_chunks(path)
result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks]
return result
def list_files(
file_store,
*,
path_prefix: str | None = None,
tags: list[str] | None = None,
metadata: dict | None = None,
limit: int = 100,
) -> dict:
"""List indexed files filtered by frontmatter exact-match, tags, and prefix.
Returns {items: [{path, metadata}], count}.
"""
metadata_filter = metadata or {}
tag_filter = tags or []
items: list[dict] = []
for path, meta in file_store.nodes.items():
if path_prefix and not path.startswith(path_prefix):
continue
md = meta.metadata or {}
if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
continue
if tag_filter:
file_tags = set(md.get("tags", []) or [])
if not all(t in file_tags for t in tag_filter):
continue
items.append({"path": path, "metadata": md})
if len(items) >= limit:
break
return {"items": items, "count": len(items)}
def _edge_to_dict(file_meta, edge) -> dict:
return {
"path": file_meta.path,
"metadata": file_meta.metadata,
"predicate": edge.predicate,
"anchor": edge.anchor,
"alias": edge.alias,
"embed": edge.embed,
"source": edge.source,
"confidence": edge.confidence,
}
def links_of(file_store, path: str) -> dict:
"""Files that `path` links TO (resolved). Each entry carries the typed-edge predicate."""
return {
"path": path,
"links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)],
}
def backlinks_of(file_store, path: str) -> dict:
"""Files that link TO `path`. Each entry carries the typed-edge predicate."""
return {
"path": path,
"backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)],
}
def wikilink_lookup(file_store, wikilink: str) -> dict:
"""Resolve a `[[target]]` wikilink with full ambiguity context.
Distinct from `file_store.resolve_wikilink(target)` (which returns
the single resolved path or None) — this surface returns the rich
payload callers need to disambiguate:
unique resolution → {wikilink, path, exists: True,
ambiguous: False, candidates: [path]}
ambiguous → {wikilink, path: None, exists: False,
ambiguous: True, candidates: [...]}
dangling → {wikilink, path: None, exists: False,
ambiguous: False, candidates: []}
"""
# Path-form (`a/b` or `a/b.md`): file_store already returns
# exactly the one path that exists, or None.
if "/" in wikilink or wikilink.endswith(".md"):
hit = file_store.resolve_wikilink(wikilink)
return {
"wikilink": wikilink,
"path": hit,
"exists": hit is not None,
"ambiguous": False,
"candidates": [hit] if hit else [],
}
# Stem-form: candidates list reveals 0/1/N resolution.
candidates = file_store.wikilink_candidates(wikilink)
if len(candidates) == 1:
return {
"wikilink": wikilink,
"path": candidates[0],
"exists": True,
"ambiguous": False,
"candidates": candidates,
}
return {
"wikilink": wikilink,
"path": None,
"exists": False,
"ambiguous": len(candidates) > 1,
"candidates": candidates,
}
async def count_tokens(
token_counter,
*,
path: str | None = None,
text: str | None = None,
) -> dict:
"""Estimate tokens for a file body (frontmatter excluded) or raw text.
Powers the Maintainer's split-trigger. Exactly one of `path` / `text`
must be provided.
"""
if path:
target = Path(path)
if not target.is_file():
return {"path": str(target), "error": "file not found"}
raw = target.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
body = post.content
tokens = await token_counter.count(messages=[], text=body)
return {
"source": "file",
"path": str(target.resolve()),
"tokens": tokens,
"body_chars": len(body),
}
if text:
tokens = await token_counter.count(messages=[], text=text)
return {
"source": "text",
"tokens": tokens,
"body_chars": len(text),
}
return {"error": "one of `path` or `text` is required"}
def iter_files(file_store) -> Iterator[tuple[str, FileMetadata]]:
"""Walk every indexed (path, FileMetadata). Used by Maintainer scans.
Equivalent to `file_store.nodes.items()`, exposed here so consumers
don't have to know about the underlying cache attribute name.
"""
return iter(file_store.nodes.items())
# ===========================================================================
# Section 3 — Projection queries (Vector / FTS / File Graph)
# ===========================================================================
#
# Per `structure.md` §"核心引擎", the Vector index, FTS5 index, and File
# Graph are downstream **projections** of the MFS — they can be wholly
# rebuilt from disk. These functions are the read entry to those
# projections; `Retriever` composes them with policy (V+K weighting,
# graph BFS, intent routing) for ranked retrieval.
async def vector_search(
file_store,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Vector similarity over the chunk-level Vector projection."""
return await file_store.vector_search(query, limit, chunk_filter)
async def keyword_search(
file_store,
query: str,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""FTS5 keyword search over the chunk-level keyword projection."""
return await file_store.keyword_search(query, limit, chunk_filter)
def expand_neighbors(
file_store,
seeds: Iterable[str],
*,
depth: int = 1,
direction: str = "both",
) -> dict[str, int]:
"""BFS over the File Graph projection. Returns {path: hop_distance}."""
return file_store.expand_neighbors(seeds, depth=depth, direction=direction)
def extract_anchors(file_store, text: str) -> list[str]:
"""Pull anchor paths from `[[target]]` references inside `text`."""
return file_store.extract_anchor_paths(text)
async def chunks_by_paths(file_store, paths: Iterable[str]) -> list[FileChunk]:
"""Batch fetch chunks across many paths (used by graph-walk retrieval)."""
return await file_store.get_chunks_by_paths(paths)
def make_chunk_filter(
file_store,
*,
paths: list[str] | None = None,
tags: list[str] | None = None,
exclude_paths: list[str] | None = None,
) -> ChunkFilter | None:
"""Build a chunk-filter against the file_store's path/tag indexes."""
return file_store.filter(paths=paths, tags=tags, exclude_paths=exclude_paths)
def all_ambiguous_wikilinks(file_store) -> dict[str, list[str]]:
"""Every stem that resolves to >1 path. Used by Maintainer.lint."""
return file_store.all_ambiguous_wikilinks()
# ===========================================================================
# Section 4 — Agent toolkit
# ===========================================================================
#
# `MemoryIO` adapts the read/write helpers above as `agentscope.tool`
# callables, with a vault-root containment check on writes and an audit
# trail every consumer can inspect after the agent's run. Used by the
# Ingestor's ReActAgent.
def _text_response(payload: Any) -> ToolResponse:
text = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2)
return ToolResponse(content=[TextBlock(type="text", text=text)])
class MemoryIO:
"""Agent-facing tool surface over the Memory File System.
Args:
file_store: The vault's FileStore (provides graph + index).
vault_root: Containment boundary — write tools refuse paths
that escape it.
Attributes:
audit: Every successful or failed write is appended here. Read
this after the agent's run to reconstruct the mutation trail.
"""
_TOOL_NAMES = (
"memory_get",
"memory_list",
"memory_resolve_wikilink",
"memory_backlinks",
"memory_links",
"memory_create",
"memory_update",
"memory_property_update",
"memory_rename",
"memory_delete",
"memory_archive",
)
def __init__(self, file_store, vault_root: str | Path):
self.file_store = file_store
self.vault_root = Path(vault_root).resolve()
self.audit: list[dict] = []
def register_all(self, toolkit: Toolkit | None = None) -> Toolkit:
"""Register every memory_* method on `toolkit` (or a fresh one)."""
toolkit = toolkit or Toolkit()
for name in self._TOOL_NAMES:
toolkit.register_tool_function(
getattr(self, name), namesake_strategy="override",
)
return toolkit
# -- Internals --------------------------------------------------------
def _resolve(self, p: str) -> Path:
pp = Path(p)
if not pp.is_absolute():
pp = self.vault_root / pp
return pp.resolve()
def _under_vault(self, p: Path) -> bool:
try:
p.relative_to(self.vault_root)
return True
except (ValueError, OSError):
return False
def _record(self, op: str, ok: bool, **fields) -> dict:
entry = {"op": op, "ok": ok, **fields}
self.audit.append(entry)
return entry
# -- Read tools (delegate to module-level helpers) --------------------
async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse:
"""Read a memory file (frontmatter + body, optional chunks).
Args:
path (str): Absolute path to the file.
include_chunks (bool): Include parsed chunk metadata.
"""
target = self._resolve(path)
result = await read_file(self.file_store, str(target), include_chunks=include_chunks)
return _text_response(result)
async def memory_list(
self,
path_prefix: str | None = None,
tags: list[str] | None = None,
metadata: dict | None = None,
limit: int = 100,
) -> ToolResponse:
"""List indexed vault files filtered by prefix, tags, and frontmatter.
Args:
path_prefix (str | None): Restrict to paths starting with this prefix.
tags (list[str] | None): All tags must be present on a file.
metadata (dict | None): Exact-match filter on frontmatter keys.
limit (int): Cap on returned items.
"""
return _text_response(list_files(
self.file_store,
path_prefix=path_prefix, tags=tags,
metadata=metadata, limit=limit,
))
async def memory_resolve_wikilink(self, wikilink: str) -> ToolResponse:
"""Resolve a `[[wikilink]]` to an absolute path.
Args:
wikilink (str): The wikilink target, e.g. `Topic` or `topics/Topic`.
"""
return _text_response(wikilink_lookup(self.file_store, wikilink))
async def memory_backlinks(self, path: str) -> ToolResponse:
"""List files linking TO a given path.
Args:
path (str): Absolute path to inspect.
"""
target = self._resolve(path)
return _text_response(backlinks_of(self.file_store, str(target)))
async def memory_links(self, path: str) -> ToolResponse:
"""List files a given path links to.
Args:
path (str): Absolute path to inspect.
"""
target = self._resolve(path)
return _text_response(links_of(self.file_store, str(target)))
# -- Write tools (delegate to write_*; record audit) ------------------
async def memory_create(
self,
path: str,
metadata: dict | None = None,
content: str = "",
overwrite: bool = False,
force: bool = False,
) -> ToolResponse:
"""Create a new markdown file in the vault.
Args:
path (str): Target path (resolved against vault_root if relative).
metadata (dict | None): YAML frontmatter for the new file.
content (str): Body markdown.
overwrite (bool): Allow overwriting an existing file.
force (bool): Allow creates that introduce stem-form wikilink ambiguity.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("create", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_create(
self.file_store, target,
metadata=dict(metadata or {}), content=content,
overwrite=overwrite, force=force,
)
entry = self._record("create", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_update(
self,
path: str,
old_string: str,
new_string: str,
replace_all: bool = False,
) -> ToolResponse:
"""Edit a file body by exact-string substitution.
Use a unique snippet for `old_string`. To append, pass the file's
tail as `old_string` and `tail + new_content` as `new_string`.
Args:
path (str): Absolute path to the file.
old_string (str): Exact text to replace (must be unique unless replace_all).
new_string (str): Replacement text.
replace_all (bool): Replace every occurrence instead of just one.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("update", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_update(target, old_string, new_string, replace_all=replace_all)
entry = self._record("update", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_property_update(self, path: str, key: str, value: Any = None) -> ToolResponse:
"""Update one YAML frontmatter key on a file (value=null deletes it).
Args:
path (str): Absolute path to the file.
key (str): Frontmatter key.
value: New value, or null to delete the key.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("property_update", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_property_update(target, key, value)
entry = self._record("property_update", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_rename(self, old_path: str, new_path: str) -> ToolResponse:
"""Rename a file and rewrite cross-vault wikilinks.
Args:
old_path (str): Current absolute path.
new_path (str): Target absolute path.
"""
old_p = self._resolve(old_path)
new_p = self._resolve(new_path)
if not self._under_vault(old_p) or not self._under_vault(new_p):
entry = self._record("rename", False, old_path=str(old_p), new_path=str(new_p),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_rename(self.file_store, self.vault_root, old_p, new_p)
entry = self._record("rename", ok, old_path=str(old_p), new_path=str(new_p), result=payload)
return _text_response(entry)
async def memory_delete(self, path: str) -> ToolResponse:
"""Delete a file from the vault.
Args:
path (str): Absolute path to the file.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("delete", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_delete(target)
entry = self._record("delete", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_archive(self, path: str, archive_dir: str = "Archive") -> ToolResponse:
"""Flip `status: archived` and move file under `<vault>/<archive_dir>/`.
Args:
path (str): Absolute path to the file.
archive_dir (str): Subdirectory name under vault_root for archives.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("archive", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_archive(self.vault_root, target, archive_dir)
entry = self._record("archive", ok, path=str(target), result=payload)
return _text_response(entry)

344
reme2/memory/retriever.py Normal file
View file

@ -0,0 +1,344 @@
"""Retriever — read service. The Memory subsystem's read-side facade.
Per the architecture blueprint, retrieval policy lives here (not in the
file_store): query understanding (intent routing) + multi-channel
fusion (V + K + graph BFS) + ranking. The file_store exposes only
single-channel index primitives; this module composes them with
strategy.
Two surfaces:
`search(query, ...)` — pure relevance retrieval (V + K hybrid).
`graph_search(query, ...)` — V + K + graph fusion (context expansion
through wikilinks).
`BaseRetriever` inherits `BaseStep` so concrete retrievers get the
standard Step lifecycle + `self.file_store` property + per-instance
configuration via constructor kwargs. The default `execute()` dispatch
just runs `graph_search` so the retriever can also be used directly as
a step inside a job pipeline; the MCP step shells in
`reme2.mcp.steps.memory_retriever` instead bypass `execute` and call
`search` / `graph_search` directly so they own the result-serialization
shape.
"""
from __future__ import annotations
import asyncio
from abc import abstractmethod
from collections import defaultdict
from ..component import R
from ..component.base_step import BaseStep
from ..schema import ChunkFilter, FileChunk
from . import memory_io
class BaseRetriever(BaseStep):
"""Pluggable retrieval strategy.
Inherits `BaseStep`, so:
- `component_type = ComponentEnum.STEP` (for registry lookup).
- `self.file_store` resolves the configured file_store via the
standard kwargs / app_context route — no manual `_start` lookup.
- kwargs (`vector_weight`, `graph_weight`, …) flow through the
standard Step kwargs handshake.
Concrete subclasses implement `search` and `graph_search`.
"""
@abstractmethod
async def search(
self,
query: str,
*,
max_results: int = 5,
min_score: float = 0.0,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Pure-relevance retrieval (V + K hybrid by convention)."""
@abstractmethod
async def graph_search(
self,
query: str = "",
*,
seeds: list[str] | None = None,
max_results: int = 5,
min_score: float = 0.0,
chunk_filter: ChunkFilter | None = None,
# Per-call overrides; None = use this retriever's own defaults.
vector_weight: float | None = None,
graph_weight: float | None = None,
graph_depth: int | None = None,
graph_decay: float | None = None,
graph_direction: str | None = None,
graph_mode: str | None = None,
graph_per_path_cap: int | None = None,
anchor_expand: bool | None = None,
) -> tuple[list[FileChunk], dict[str, int]]:
"""Three-way fusion search (V + K + graph BFS over wikilinks).
Returns:
(chunks, hops) — `hops[path]` is the BFS distance from the
seed set to that path (0 = seed, 1 = 1-hop neighbor, …).
"""
async def execute(self):
"""Default Step entry point — dispatches to `graph_search`.
The retriever is normally invoked via `search` / `graph_search`
directly by the MCP step shells, but this lets the retriever
also be wired as a step inside a job pipeline (e.g. for
debugging / scripts). Reads RuntimeContext for the standard
query / max_results / min_score / seeds + filter args.
"""
assert self.context is not None
ctx = self.context
chunk_filter = memory_io.make_chunk_filter(
self.file_store,
paths=ctx.get("paths") or None,
tags=ctx.get("tags") or None,
exclude_paths=ctx.get("exclude_paths") or None,
)
results, _hops = await self.graph_search(
query=ctx.get("query", "").strip(),
seeds=list(ctx.get("seeds") or []),
max_results=int(ctx.get("max_results", 5)),
min_score=float(ctx.get("min_score", 0.0)),
chunk_filter=chunk_filter,
)
ctx.response.success = True
ctx.response.answer = [c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results]
@R.register("hybrid")
class HybridRetriever(BaseRetriever):
"""V + K (+ optional graph BFS) fusion retriever.
Composes the file_store's single-channel primitives (`vector_search`,
`keyword_search`, `expand_neighbors`, `extract_anchor_paths`,
`get_chunks_by_paths`). Knobs (`vector_weight`, `graph_weight`,
`graph_depth`, `graph_decay`, `graph_direction`, `graph_mode`,
`graph_per_path_cap`, `anchor_expand`, `candidate_multiplier`) are
constructor defaults; `graph_search` accepts per-call overrides for
the graph fusion knobs (any None → fall back to constructor default).
"""
def __init__(
self,
vector_weight: float = 0.7,
graph_weight: float = 0.3,
graph_depth: int = 1,
graph_decay: float = 0.5,
graph_direction: str = "both",
graph_mode: str = "additive",
graph_per_path_cap: int = 3,
candidate_multiplier: float = 3.0,
anchor_expand: bool = True,
**kwargs,
):
super().__init__(**kwargs)
if graph_mode not in ("additive", "boost"):
raise ValueError(f"graph_mode must be 'additive' or 'boost', got {graph_mode!r}")
self.vector_weight = vector_weight
self.graph_weight = graph_weight
self.graph_depth = graph_depth
self.graph_decay = graph_decay
self.graph_direction = graph_direction
self.graph_mode = graph_mode
self.graph_per_path_cap = graph_per_path_cap
self.candidate_multiplier = candidate_multiplier
self.anchor_expand = anchor_expand
async def search(
self,
query: str,
*,
max_results: int = 5,
min_score: float = 0.0,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""V + K hybrid retrieval (no graph). Owns the fusion policy.
Pipeline:
1. If both channels are enabled → run V + K in parallel,
merge by unique_key with weighted score.
2. If only one is enabled → return that channel's results
directly (no fusion to do).
3. If neither → empty.
"""
fs = self.file_store
candidates = min(200, max(1, int(max_results * self.candidate_multiplier)))
text_weight = 1.0 - self.vector_weight
if fs.vector_enabled and fs.fts_enabled:
v_task = memory_io.vector_search(fs, query, candidates, chunk_filter)
k_task = memory_io.keyword_search(fs, query, candidates, chunk_filter)
v_results, k_results = await asyncio.gather(v_task, k_task)
if not k_results:
results = v_results[:max_results]
elif not v_results:
results = k_results[:max_results]
else:
results = self._merge_vk(
v_results, k_results, self.vector_weight, text_weight,
)[:max_results]
elif fs.vector_enabled:
results = await memory_io.vector_search(fs, query, max_results, chunk_filter)
elif fs.fts_enabled:
results = await memory_io.keyword_search(fs, query, max_results, chunk_filter)
else:
results = []
if min_score > 0:
results = [r for r in results if r.score >= min_score]
return results
@staticmethod
def _merge_vk(
vector: list[FileChunk],
keyword: list[FileChunk],
vector_weight: float,
text_weight: float,
) -> list[FileChunk]:
"""Weighted V+K merge by unique_key. Vector first (canonical),
keyword adds its weighted subscore on collision or seeds fresh."""
merged: dict[str, FileChunk] = {}
for r in vector:
r.scores["score"] = r.scores.get("vector", 0.0) * vector_weight
merged[r.unique_key] = r
for r in keyword:
key = r.unique_key
k = r.scores.get("keyword", 0.0)
if key in merged:
merged[key].scores["score"] += k * text_weight
else:
r.scores["score"] = k * text_weight
merged[key] = r
results = list(merged.values())
results.sort(key=lambda c: c.score, reverse=True)
return results
async def graph_search(
self,
query: str = "",
*,
seeds: list[str] | None = None,
max_results: int = 5,
min_score: float = 0.0,
chunk_filter: ChunkFilter | None = None,
vector_weight: float | None = None,
graph_weight: float | None = None,
graph_depth: int | None = None,
graph_decay: float | None = None,
graph_direction: str | None = None,
graph_mode: str | None = None,
graph_per_path_cap: int | None = None,
anchor_expand: bool | None = None,
) -> tuple[list[FileChunk], dict[str, int]]:
# Resolve per-call overrides → constructor defaults.
vw = self.vector_weight if vector_weight is None else float(vector_weight)
gw = self.graph_weight if graph_weight is None else float(graph_weight)
gd = self.graph_depth if graph_depth is None else int(graph_depth)
gdc = self.graph_decay if graph_decay is None else float(graph_decay)
gdr = self.graph_direction if graph_direction is None else graph_direction
gm = self.graph_mode if graph_mode is None else graph_mode
gpc = self.graph_per_path_cap if graph_per_path_cap is None else int(graph_per_path_cap)
ae = self.anchor_expand if anchor_expand is None else bool(anchor_expand)
if gm not in ("additive", "boost"):
raise ValueError(f"graph_mode must be 'additive' or 'boost', got {gm!r}")
if not (0.0 <= vw <= 1.0):
raise ValueError(f"vector_weight must be in [0,1], got {vw}")
if not (0.0 <= gw <= 1.0):
raise ValueError(f"graph_weight must be in [0,1], got {gw}")
explicit_seeds = list(seeds or [])
if not query and not explicit_seeds:
raise ValueError("graph_search: query or seeds must be provided")
candidate_count = max(max_results, int(max_results * self.candidate_multiplier))
fs = self.file_store
# 1. V + K in parallel (each is a no-op when its backend is disabled).
if query:
v_task = memory_io.vector_search(fs, query, candidate_count, chunk_filter)
k_task = memory_io.keyword_search(fs, query, candidate_count, chunk_filter)
v_results, k_results = await asyncio.gather(v_task, k_task)
else:
v_results, k_results = [], []
# 2. Build seed set.
seed_paths: set[str] = set()
for c in v_results:
seed_paths.add(c.path)
for c in k_results:
seed_paths.add(c.path)
for p in explicit_seeds:
if p in fs:
seed_paths.add(p)
if ae and query:
seed_paths.update(memory_io.extract_anchors(fs, query))
# 3. Graph expansion.
if gw > 0 and seed_paths and gd >= 0:
hops = memory_io.expand_neighbors(fs, seed_paths, depth=gd, direction=gdr)
else:
hops = {}
graph_scores: dict[str, float] = {p: gdc ** h for p, h in hops.items()}
# 4. Pull graph-only chunks (paths in expansion but not in V∪K).
# Skip in 'boost' mode — boost only re-ranks V∪K, never adds candidates.
vk_paths = {c.path for c in v_results} | {c.path for c in k_results}
if gm == "additive" and hops:
extra_paths = set(hops) - vk_paths
if chunk_filter is not None and chunk_filter.resolved_paths is not None:
extra_paths &= chunk_filter.resolved_paths
extra_chunks = await memory_io.chunks_by_paths(fs, extra_paths)
# Per-path cap so a hub topic with N chunks doesn't flood results.
by_path: dict[str, list] = defaultdict(list)
for c in extra_chunks:
by_path[c.path].append(c)
graph_only_chunks: list[FileChunk] = []
for path_chunks in by_path.values():
path_chunks.sort(key=lambda c: c.start_line)
graph_only_chunks.extend(path_chunks[:gpc])
else:
graph_only_chunks = []
# 5. Merge by unique_key. Vector first (canonical), keyword fills its
# subscore on existing entries, graph-only chunks come in fresh.
pool: dict = {}
for c in v_results:
pool[c.unique_key] = c
for c in k_results:
existing = pool.get(c.unique_key)
if existing is None:
pool[c.unique_key] = c
else:
existing.scores["keyword"] = c.scores.get("keyword", 0.0)
for c in graph_only_chunks:
pool.setdefault(c.unique_key, c)
# 6. Final scoring.
for c in pool.values():
v = c.scores.get("vector", 0.0)
k = c.scores.get("keyword", 0.0)
g = graph_scores.get(c.path, 0.0)
vk = vw * v + (1.0 - vw) * k
if gm == "boost":
final = vk * (1.0 + gw * g)
else:
final = (1.0 - gw) * vk + gw * g
c.scores["graph"] = g
c.scores["score"] = final
# 7. Rank + filter + slice.
results = sorted(pool.values(), key=lambda c: c.score, reverse=True)
if min_score > 0:
results = [r for r in results if r.score >= min_score]
results = results[:max_results]
return results, hops

28
reme2/file_based/summarizer.py → reme2/memory/summarizer.py
View file

@ -18,16 +18,16 @@ class Summarizer(BaseStep):
"""Summarizer step for summarizing memory messages."""
def __init__(
self,
working_dir: str,
memory_dir: str,
memory_compact_threshold: int,
toolkit: Toolkit | None = None,
console_enabled: bool = False,
timezone: str | None = None,
add_thinking_block: bool = True,
as_token_counter: HuggingFaceTokenCounter | None = None,
**kwargs,
self,
working_dir: str,
memory_dir: str,
memory_compact_threshold: int,
toolkit: Toolkit | None = None,
console_enabled: bool = False,
timezone: str | None = None,
add_thinking_block: bool = True,
as_token_counter: HuggingFaceTokenCounter | None = None,
**kwargs,
):
"""Initialize the summarizer step.
@ -227,10 +227,10 @@ class Summarizer(BaseStep):
return total
async def _format_msgs_to_str(
self,
messages: list[Msg],
memory_compact_threshold: int,
include_thinking: bool = True,
self,
messages: list[Msg],
memory_compact_threshold: int,
include_thinking: bool = True,
) -> str:
"""Format list of messages to a single formatted string.

20
reme2/file_based/summarizer.yaml → reme2/memory/summarizer.yaml
View file

@ -1,14 +1,14 @@
user_message: |
Memory Pre-compression Flush Cycle.
The current session is about to enter the automatic compression phase. Please capture persistent memory AND session reflections, then write them to disk.
Current date: {date}
Working directory: {working_dir}
# Task
Immediately store persistent memory and reflections to: {memory_dir}/YYYY-MM-DD.md
# Workflow
1. Extract and synthesize content from the current session:
- Persistent Memory: Facts, user profile updates, project states, and important events.
@ -17,7 +17,7 @@ user_message: |
- If the file doesn’t exist, use `write` tool directly.
- If the file exists, intelligently merge new information with existing content, prefer using `edit` to update specific sections.
- Use `write` to overwrite the entire file only if substantial restructuring is required.
# Principles
- Intelligently merge new information with existing content:
- Categorize clearly (e.g., separate "Factual Memory" from "Reflections & Logic").
@ -32,15 +32,15 @@ user_message: |
user_message_zh: |
预压缩内存刷新轮次。
当前会话即将进入自动压缩阶段；请将持久化记忆与经验反思捕获并写入磁盘。
当前日期：{date}
工作目录：{working_dir}
# 任务
立即存储持久化记忆与反思（使用路径 {memory_dir}/YYYY-MM-DD.md）。
# 工作流程
1. 从当前会话中提取并综合两类内容：
- 持久化记忆：客观事实、用户信息更新、项目状态及重要事件。
@ -48,7 +48,7 @@ user_message_zh: |
2. `read` {memory_dir}/YYYY-MM-DD.md（如文件不存在，会返回错误提示）
- 若文件不存在，直接使用 `write` 工具写入。
- 若文件已存在，智能合并新信息与现有内容，尽可能使用 `edit` 更新特定部分，仅在需要大幅重构时使用 `write` 覆盖整个文件。
# 原则
- 智能合并新信息与现有内容：
- 将内容进行清晰的分类（例如明确区分“事实记忆”与“反思与逻辑”）。

54
reme2/reme_backup.py
View file

@ -13,7 +13,7 @@ from .application import Application
from .component import R, RuntimeContext
from .config import parse_args
from .enumeration import ComponentEnum
from .file_based.summarizer import Summarizer
from .memory.summarizer import Summarizer
from .utils import run_coro_safely
@ -21,17 +21,17 @@ class ReMe(Application):
"""ReMe memory management application."""
async def summarize(
self,
messages: list[Msg],
as_llm: str | ChatModelBase = "default",
as_llm_formatter: str | FormatterBase = "default",
as_token_counter: str | TokenCounterBase | HuggingFaceTokenCounter = "default",
toolkit: Toolkit | None = None,
language: str = "zh",
max_input_length: float = 128 * 1024,
compact_ratio: float = 0.7,
timezone: str | None = None,
add_thinking_block: bool = True,
self,
messages: list[Msg],
as_llm: str | ChatModelBase = "default",
as_llm_formatter: str | FormatterBase = "default",
as_token_counter: str | TokenCounterBase | HuggingFaceTokenCounter = "default",
toolkit: Toolkit | None = None,
language: str = "zh",
max_input_length: float = 128 * 1024,
compact_ratio: float = 0.7,
timezone: str | None = None,
add_thinking_block: bool = True,
) -> str:
"""Summarize and compact memory messages.
@ -84,7 +84,7 @@ class ReMe(Application):
async def memory_search(self, query: str, max_results: int = 5, min_score: float = 0.1) -> str:
"""Search memory for relevant entries."""
from .file_based.memory_search import MemorySearch
from .memory.memory_retriever import MemorySearch
try:
search_step = MemorySearch()
@ -94,25 +94,25 @@ class ReMe(Application):
return str(e)
async def dream(
self,
as_llm: str | ChatModelBase = "default",
as_llm_formatter: str | FormatterBase = "default",
as_token_counter: str | TokenCounterBase = "default",
toolkit: Toolkit | None = None,
language: str = "zh",
timezone: str | None = None,
self,
as_llm: str | ChatModelBase = "default",
as_llm_formatter: str | FormatterBase = "default",
as_token_counter: str | TokenCounterBase = "default",
toolkit: Toolkit | None = None,
language: str = "zh",
timezone: str | None = None,
) -> str:
"""Process and consolidate memories in background."""
return ""
async def proactive(
self,
as_llm: str | ChatModelBase = "default",
as_llm_formatter: str | FormatterBase = "default",
as_token_counter: str | TokenCounterBase = "default",
toolkit: Toolkit | None = None,
language: str = "zh",
timezone: str | None = None,
self,
as_llm: str | ChatModelBase = "default",
as_llm_formatter: str | FormatterBase = "default",
as_token_counter: str | TokenCounterBase = "default",
toolkit: Toolkit | None = None,
language: str = "zh",
timezone: str | None = None,
) -> str:
"""Generate proactive memory insights."""
return ""

4
reme2/schema/__init__.py
View file

@ -5,7 +5,9 @@ from .as_msg_stat import AsBlockStat, AsMsgStat
from .base_node import BaseNode
from .chunk_filter import ChunkFilter
from .file_chunk import FileChunk
from .file_edge import FileEdge
from .file_metadata import FileMetadata
from .parsed_file import ParsedFile
from .request import Request
from .response import Response
from .stream_chunk import StreamChunk
@ -19,7 +21,9 @@ __all__ = [
"BaseNode",
"ChunkFilter",
"FileChunk",
"FileEdge",
"FileMetadata",
"ParsedFile",
"Request",
"Response",
"StreamChunk",

14
reme2/schema/chunk_filter.py
View file

@ -1,23 +1,23 @@
"""Filter for chunk-store search.
"""Filter for chunk search.
User-facing fields (paths/tags/exclude_paths) describe metadata-level intent.
FileGraph compiles them into `resolved_paths` (the concrete path set that
ChunkStore actually consumes for filtering).
file_store compiles them into `resolved_paths` (the concrete path set that
the chunk search actually consumes for filtering).
"""
from pydantic import BaseModel, Field
class ChunkFilter(BaseModel):
"""User-facing search filter, resolved by FileGraph into a path set.
"""User-facing search filter, resolved by file_store into a path set.
User input fields:
paths: include only paths starting with any of these prefixes
tags: include only files whose metadata contains ALL these tags
exclude_paths: exclude paths starting with any of these prefixes
Compiled field (set by FileGraph.filter):
resolved_paths: concrete path set to restrict ChunkStore search.
Compiled field (set by file_store.filter):
resolved_paths: concrete path set to restrict chunk search.
None = no restriction (all chunks).
Empty set = no chunks match (search returns empty).
"""
@ -43,7 +43,7 @@ class ChunkFilter(BaseModel):
return True
def match_path(self, path: str) -> bool:
"""Match a path against the resolved path set (used by ChunkStore)."""
"""Match a path against the resolved path set (used by FileStore)."""
if self.resolved_paths is None:
return True
return path in self.resolved_paths

43
reme2/schema/file_edge.py Normal file
View file

@ -0,0 +1,43 @@
"""FileEdge — typed wikilink edge between vault files.
Replaces the prior `link: list[str]` model on FileMetadata. An edge
captures both the bare wikilink shape (`target` + optional anchor /
alias / embed prefix) AND the typed-edge predicate that turns a plain
`[[X]]` reference into a graph relation.
Sources:
"regex" — extracted from inline syntax (`[[X]]`, `[pred:: [[X]]]`)
"frontmatter" — inferred from a frontmatter key acting as predicate
(e.g. `author: "[[John]]"` → predicate="author")
"llm" — produced by an IE pipeline; should set `confidence`
The `target` field stays raw (e.g. `"X"` or `"topics/X"`) — resolution
to an absolute path is done by the file_store via `resolve_wikilink`.
"""
from typing import Literal
from pydantic import BaseModel, Field
class FileEdge(BaseModel):
target: str = Field(..., description="Raw wikilink target as written in source.")
predicate: str | None = Field(
default=None,
description="Typed-edge predicate (Dataview-style). None for plain wikilinks.",
)
anchor: str | None = Field(default=None, description="Heading or block anchor (after #).")
alias: str | None = Field(default=None, description="Display alias (after |).")
embed: bool = Field(default=False, description="True for `![[X]]` embed prefix.")
source: Literal["regex", "frontmatter", "llm"] = Field(
default="regex",
description="Provenance of this edge.",
)
confidence: float | None = Field(
default=None,
description="LLM confidence (0..1). None for regex/frontmatter.",
)
@property
def is_typed(self) -> bool:
return self.predicate is not None

119
reme2/schema/file_graph.py
View file

@ -1,119 +0,0 @@
import json
from collections import defaultdict
from pathlib import Path
from .chunk_filter import ChunkFilter
from .file_metadata import FileMetadata
class FileGraph:
def __init__(self) -> None:
self._nodes: dict[str, FileMetadata] = {}
self._backlinks: dict[str, set[str]] = defaultdict(set)
# -- CRUD ----------------------------------------------------------------
def create(self, *metadatas: FileMetadata) -> None:
for metadata in metadatas:
path = metadata.path
if path in self._nodes:
self._remove_forward(self._nodes[path])
self._nodes[path] = metadata
for metadata in metadatas:
self._add_forward(metadata)
def read(self, path: str) -> FileMetadata | None:
return self._nodes.get(path)
def update(self, path: str, **fields) -> FileMetadata | None:
metadata = self._nodes.get(path)
if metadata is None:
return None
self._remove_forward(metadata)
updated = metadata.model_copy(update=fields)
self._nodes[path] = updated
self._add_forward(updated)
return updated
def delete(self, path: str) -> FileMetadata | None:
metadata = self._nodes.pop(path, None)
if metadata is None:
return None
self._remove_forward(metadata)
self._backlinks.pop(path, None)
return metadata
# -- Link queries --------------------------------------------------------
def get_links(self, path: str) -> list[FileMetadata]:
metadata = self._nodes.get(path)
if metadata is None:
return []
return [self._nodes[link] for link in metadata.link if link in self._nodes]
def get_backlinks(self, path: str) -> list[FileMetadata]:
return [
self._nodes[src]
for src in self._backlinks.get(path, set())
if src in self._nodes
]
def filter(
self,
paths: list[str] | None = None,
tags: list[str] | None = None,
exclude_paths: list[str] | None = None,
) -> ChunkFilter:
cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
if cf.is_empty():
return cf
cf.resolved_paths = {
path for path, meta in self._nodes.items()
if cf.match_metadata(path, meta.metadata)
}
return cf
# -- Index helpers -------------------------------------------------------
def _add_forward(self, metadata: FileMetadata) -> None:
for link in metadata.link:
self._backlinks[link].add(metadata.path)
def _remove_forward(self, metadata: FileMetadata) -> None:
for link in metadata.link:
self._backlinks[link].discard(metadata.path)
# -- Persistence ---------------------------------------------------------
def save(self, path: str | Path) -> None:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
raw = {p: meta.model_dump(mode="json") for p, meta in self._nodes.items()}
content = json.dumps(raw, ensure_ascii=False)
temp = path.with_suffix(".tmp")
temp.write_text(content, encoding="utf-8")
temp.replace(path)
@classmethod
def load(cls, path: str | Path) -> "FileGraph":
path = Path(path)
graph = cls()
if not path.exists():
return graph
raw: dict = json.loads(path.read_text(encoding="utf-8"))
nodes = [FileMetadata(**meta) for meta in raw.values()]
graph.create(*nodes)
return graph
# -- Dunder --------------------------------------------------------------
@property
def nodes(self) -> dict[str, FileMetadata]:
return self._nodes
def __len__(self) -> int:
return len(self._nodes)
def __contains__(self, path: str) -> bool:
return path in self._nodes

1
reme2/schema/file_metadata.py
View file

@ -5,5 +5,4 @@ class FileMetadata(BaseModel):
file: str = Field(...)
path: str = Field(...)
st_mtime: float = Field(...)
link: list[str] = Field(default_factory=list)
metadata: dict = Field(default_factory=dict)

28
reme2/schema/parsed_file.py Normal file
View file

@ -0,0 +1,28 @@
"""ParsedFile — Parser's single output, carrying everything one file pass produces.
Inherits `FileMetadata` (file / path / st_mtime / metadata) and adds the
parsed contents:
chunks: list[FileChunk] — semantic blocks (text + position + hash);
embeddings ARE attached here — the parser
owns the embedding step (with hash-diff
cache via `existing_chunks` parameter).
edges: list[FileEdge] — wikilink graph edges (regex / frontmatter /
LLM provenance preserved per-edge).
The Watcher passes a `ParsedFile` into `file_store.upsert_parsed(...)`,
which dispatches the three sub-payloads (meta / edges / chunks) to the
appropriate persistence pipelines while keeping the upsert atomic at
the store layer.
"""
from pydantic import Field
from .file_chunk import FileChunk
from .file_edge import FileEdge
from .file_metadata import FileMetadata
class ParsedFile(FileMetadata):
chunks: list[FileChunk] = Field(default_factory=list)
edges: list[FileEdge] = Field(default_factory=list)

41
reme2/schema/vault/__init__.py Normal file
View file

@ -0,0 +1,41 @@
"""Vault business schemas — domain models for the markdown vault.
These are the *business objects* the user stores in the vault (Topic,
Event), distinct from the engine schemas in `reme2/schema/` (FileMetadata,
FileChunk, ChunkFilter — internal data types).
Single Topic class covers all categories under topics/ (no satellite /
tentacle subclassing); folder-topic is identified by path convention.
Lives under `reme2/schema/vault/` (not in `reme2/mcp/`) so memory
services (Maintainer.lint, Ingestor) can validate frontmatter without
importing the MCP transport layer — that's what was creating the
mcp ↔ memory dependency cycle.
"""
from .event import Event, EventStatus
from .frontmatter import VaultBaseFrontmatter, parse_frontmatter
from .registry import ALL_KNOWN_CATEGORIES, schema_for
from .topic import (
INDEX_CATEGORIES,
JUDGMENT_CATEGORIES,
CONTENT_CATEGORIES,
Confidence,
Topic,
TopicCategory,
)
__all__ = [
"ALL_KNOWN_CATEGORIES",
"CONTENT_CATEGORIES",
"Confidence",
"Event",
"EventStatus",
"INDEX_CATEGORIES",
"JUDGMENT_CATEGORIES",
"Topic",
"TopicCategory",
"VaultBaseFrontmatter",
"parse_frontmatter",
"schema_for",
]

16
reme2/schema/vault/event.py Normal file
View file

@ -0,0 +1,16 @@
"""Event schema — task process records under events/{date}/{name}/."""
from typing import Literal
from .frontmatter import VaultBaseFrontmatter
EventStatus = Literal["active", "distilled", "archived"]
class Event(VaultBaseFrontmatter):
"""events/{YYYY-MM-DD}/{name}/{name}.md."""
category: Literal["event"] = "event" # type: ignore[assignment]
status: EventStatus = "active"
topics: list[str] = []
originSessionId: str | None = None

38
reme2/schema/vault/frontmatter.py Normal file
View file

@ -0,0 +1,38 @@
"""Common frontmatter schema shared by all vault files."""
from datetime import date, datetime
from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_validator
class VaultBaseFrontmatter(BaseModel):
"""Fields present on every vault file."""
model_config = ConfigDict(extra="allow")
title: str = Field(...)
description: str = Field(default="")
tags: list[str] = Field(default_factory=list)
category: str = Field(...)
created: date | None = None
updated: date | None = None
@field_validator("created", "updated", mode="before")
@classmethod
def _coerce_date(cls, v):
if v is None or isinstance(v, date):
return v
if isinstance(v, datetime):
return v.date()
if isinstance(v, str):
return datetime.fromisoformat(v).date()
return v
def parse_frontmatter(raw: dict[str, Any]) -> VaultBaseFrontmatter:
"""Tolerant parse — never raises; missing required fields fall back to defaults."""
safe = dict(raw or {})
safe.setdefault("title", safe.get("name") or "")
safe.setdefault("category", "unknown")
return VaultBaseFrontmatter.model_validate(safe)

27
reme2/schema/vault/registry.py Normal file
View file

@ -0,0 +1,27 @@
"""Map vault category → strict pydantic model for validation.
Used by the Maintainer to derive lint rules from schema instead of
hardcoding. Two business object schemas:
- event → Event
- everything else → Topic (judgment subset enforced via model_validator)
"""
from typing import get_args
from pydantic import BaseModel
from .event import Event
from .topic import Topic, TopicCategory
_REGISTRY: dict[str, type[BaseModel]] = {"event": Event}
for _cat in get_args(TopicCategory):
_REGISTRY[_cat] = Topic
ALL_KNOWN_CATEGORIES: frozenset[str] = frozenset(_REGISTRY.keys())
def schema_for(category: str | None) -> type[BaseModel] | None:
"""Return the pydantic schema bound to a category, or None for unknown."""
if not category:
return None
return _REGISTRY.get(category)

47
reme2/schema/vault/topic.py Normal file
View file

@ -0,0 +1,47 @@
"""Topic schema — every .md under topics/ is a Topic.
No satellite/tentacle/folder note subclassing. Folder topic = Topic whose
filename equals its parent directory name (path convention; not a schema field).
"""
from typing import Literal
from pydantic import model_validator
from .frontmatter import VaultBaseFrontmatter
# Topic categories grouped by usage:
# - Index categories: usually appear as folder topics; may have ticker/market
# - Content categories: cluster siblings; thesis/model/questions need confidence
INDEX_CATEGORIES = {"company", "sector", "concept", "method", "tool", "profile"}
JUDGMENT_CATEGORIES = {"thesis", "model", "questions"}
CONTENT_CATEGORIES = JUDGMENT_CATEGORIES | {"fundamentals"}
TopicCategory = Literal[
"company", "sector", "concept", "method", "tool", "profile",
"thesis", "model", "questions", "fundamentals",
]
Confidence = Literal["⏳", "✅", "❌"]
class Topic(VaultBaseFrontmatter):
"""topics/{folder}/{name}.md — long-lived cognitive memory node.
Folder topic is identified by path convention: filename stem == parent
folder name. Not represented in this schema directly; checked at runtime
via `Path(p).stem == Path(p).parent.name`.
"""
category: TopicCategory # type: ignore[assignment]
market: str | None = None
ticker: str | None = None
confidence: Confidence | None = None
@model_validator(mode="after")
def _confidence_required_for_judgments(self):
if self.category in JUDGMENT_CATEGORIES and self.confidence is None:
raise ValueError(
f"category={self.category} requires explicit confidence "
f"(one of ⏳ / ✅ / ❌)"
)
return self

18
reme2/utils/__init__.py
View file

@ -1,17 +1,24 @@
"""Utility modules"""
from .case_converter import camel_to_snake, snake_to_camel
from .chunking_utils import chunk_markdown
from .common_utils import hash_text, execute_stream_task, run_coro_safely
from .logger_utils import get_logger
from .logo_utils import print_logo
from .similarity_utils import cosine_similarity, batch_cosine_similarity
from .singleton import singleton
from .wikilink import (
InlineField,
extract_inline_fields,
extract_typed_edges,
extract_wikilinks,
extract_wikilinks_from_metadata,
parse_wikilinks,
parse_wikilinks_from_metadata,
)
__all__ = [
"camel_to_snake",
"snake_to_camel",
"chunk_markdown",
"hash_text",
"execute_stream_task",
"run_coro_safely",
@ -20,4 +27,11 @@ __all__ = [
"cosine_similarity",
"batch_cosine_similarity",
"singleton",
"InlineField",
"extract_wikilinks",
"extract_wikilinks_from_metadata",
"parse_wikilinks",
"parse_wikilinks_from_metadata",
"extract_inline_fields",
"extract_typed_edges",
]

144
reme2/utils/chunking_utils.py
View file

@ -1,144 +0,0 @@
"""Markdown file chunking utilities.
Provides functionality to split Markdown documents into smaller chunks
while maintaining overlap between consecutive chunks for context preservation.
"""
from .common_utils import hash_text
from ..schema import FileChunk
def chunk_markdown(
text: str,
path: str,
chunk_tokens: int,
overlap: int,
) -> list[FileChunk]:
"""Split Markdown text into chunks with configurable size and overlap.
Implements a sliding window approach to chunk Markdown content while
preserving context through overlap between consecutive chunks. Token
counts are approximated using a 1:4 ratio (1 token ≈ 4 characters).
Args:
text: Input Markdown text to be chunked.
path: File path identifier for the source document.
chunk_tokens: Maximum number of tokens per chunk. Will be converted
to characters using the 1:4 ratio, with a minimum of 32 characters.
overlap: Number of overlapping tokens between consecutive chunks.
Helps maintain context across chunk boundaries.
Returns:
A list of FileChunk objects, each containing:
- id: Unique identifier based on path, line numbers, and hash
- path: The source file path
- start_line: Starting line number (1-indexed)
- end_line: Ending line number (1-indexed)
- text: The chunk content
- hash: SHA-256 hash of the chunk content
Examples:
>>> text = "# Header\\nParagraph content here.\\n\\n## Subheader"
>>> chunks = chunk_markdown(text, "doc.md", chunk_tokens=100, overlap=20)
>>> len(chunks)
1
>>> chunks[0].path
'doc.md'
"""
if not text.strip():
return []
lines = text.split("\n")
# Convert tokens to characters (~1 token = 4 chars)
max_chars = max(32, chunk_tokens * 4)
overlap_chars = max(0, overlap * 4)
chunks: list[FileChunk] = []
# Currently building chunk
current: list[dict] = [] # [{'line': str, 'line_no': int}]
current_chars = 0
def flush() -> None:
"""Add current chunk to results list."""
if not current:
return
first_entry = current[0]
last_entry = current[-1]
if not first_entry or not last_entry:
return
chunk_text = "\n".join([entry["line"] for entry in current])
start_line = first_entry["line_no"]
end_line = last_entry["line_no"]
chunk_hash = hash_text(chunk_text)
chunks.append(
FileChunk(
id=hash_text(f"{path}:{start_line}:{end_line}:{chunk_hash}:{len(chunks)}"),
path=path,
start_line=start_line,
end_line=end_line,
text=chunk_text,
hash=chunk_hash,
),
)
def carry_overlap() -> None:
"""Keep overlapping part and clear the rest."""
nonlocal current, current_chars
if overlap_chars <= 0 or not current:
current = []
current_chars = 0
return
acc = 0
kept = []
# Collect lines from the end until reaching overlap size
for j in range(len(current) - 1, -1, -1):
entry = current[j]
if not entry:
continue
acc += len(entry["line"]) + 1 # +1 for newline
kept.insert(0, entry) # Insert at the beginning to maintain order
if acc >= overlap_chars:
break
current = kept
current_chars = sum(len(entry["line"]) + 1 for entry in kept)
for i, line in enumerate(lines):
line_no = i + 1
# Split long lines into multiple segments
segments = []
if not line: # Empty line
segments.append("")
else:
# If line is too long, split by maximum character count
for start in range(0, len(line), max_chars):
segments.append(line[start: start + max_chars])
for segment in segments:
line_size = len(segment) + 1 # +1 for newline
# If adding current segment would exceed the limit, flush current chunk
if current_chars + line_size > max_chars and current:
flush()
carry_overlap()
current.append({"line": segment, "line_no": line_no})
current_chars += line_size
# Process the final chunk
flush()
return [c for c in chunks if c.text.strip()]

8
reme2/utils/common_utils.py
View file

@ -47,10 +47,10 @@ def hash_text(text: str, encoding: str = "utf-8") -> str:
async def execute_stream_task(
stream_queue: asyncio.Queue,
task: asyncio.Task,
task_name: str | None = None,
output_format: Literal["str", "bytes", "chunk"] = "str",
stream_queue: asyncio.Queue,
task: asyncio.Task,
task_name: str | None = None,
output_format: Literal["str", "bytes", "chunk"] = "str",
) -> AsyncGenerator[str | bytes | StreamChunk, None]:
"""Core stream flow execution logic.

10
reme2/utils/logger_utils.py
View file

@ -14,11 +14,11 @@ _initialized = False
def get_logger(
log_dir: str = "logs",
level: str = "INFO",
log_to_console: bool = True,
log_to_file: bool = True,
force_init: bool = False,
log_dir: str = "logs",
level: str = "INFO",
log_to_console: bool = True,
log_to_file: bool = True,
force_init: bool = False,
):
"""Get a configured logger instance.

78
reme2/utils/vault_paths.py Normal file
View file

@ -0,0 +1,78 @@
"""Vault path generation + naming disambiguation.
Pure-Python helpers consumed by the MCP step shells (`sync`,
`topic_create`) and any future flow that needs to materialize a path
under the vault layout. No async, no MCP awareness — easy to unit-test.
Wikilink uniqueness is a *graph* property, not a path-builder concern —
see `reme2.component.file_store.BaseFileStore.collisions_after_create`
and `BaseFileStore.all_ambiguous_wikilinks`.
"""
from __future__ import annotations
import re
from collections.abc import Iterable
from datetime import date as date_type
from pathlib import Path
def event_path(
vault_root: str | Path,
name: str,
on_date: date_type | str | None = None,
events_dir: str = "events",
) -> Path:
"""events/{YYYY-MM-DD}/{name}/{name}.md under the given vault root."""
if on_date is None:
on_date = date_type.today()
if isinstance(on_date, date_type):
on_date = on_date.isoformat()
return Path(vault_root) / events_dir / on_date / name / f"{name}.md"
def topic_path(
vault_root: str | Path,
folder: str,
name: str,
topics_dir: str = "topics",
) -> Path:
"""topics/{folder}/{name}.md under the given vault root."""
return Path(vault_root) / topics_dir / folder / f"{name}.md"
def is_folder_topic(path: str | Path) -> bool:
"""True if filename stem == parent directory name (folder note convention)."""
p = Path(path)
return p.stem == p.parent.name
def next_suffixed_stem(taken: Iterable[str], base: str) -> str:
"""Lowest unused `<base>-N` (N≥2). Returns `base` itself if not taken.
Used by topic_create / sync when a same-stem (topic) or
same-name-on-same-day (event) collision is detected — the suggested
suffix is *advisory*; the create still rejects so the agent can pick
a domain-specific qualifier (e.g., `Apple-Inc` vs `Apple-Fruit`)
that carries more meaning than a numeric suffix.
Examples (with taken={"BABA", "BABA-2"}):
next_suffixed_stem(taken, "BABA") == "BABA-3"
next_suffixed_stem(taken, "MSFT") == "MSFT"
"""
taken_set = set(taken)
if base not in taken_set:
return base
pattern = re.compile(rf"^{re.escape(base)}-(\d+)$")
used: set[int] = set()
for s in taken_set:
m = pattern.match(s)
if m:
try:
used.add(int(m.group(1)))
except ValueError:
continue
n = 2
while n in used:
n += 1
return f"{base}-{n}"

259
reme2/utils/wikilink.py Normal file
View file

@ -0,0 +1,259 @@
"""Wikilink protocol — Obsidian-compatible bare links + Dataview-style typed edges.
## Bare wikilinks (Obsidian)
[[Note]] short form (resolve by stem)
[[path/to/Note]] path form
[[Note#Section]] heading anchor
[[Note#^block-id]] block anchor
[[Note|Alias]] display alias
![[Note]] embed (transclusion)
![[Note#Section|Alias]] all combined
## Typed wikilinks / property edges (Dataview inline field syntax)
[predicate:: [[Target]]] typed link (bracketed, visible)
(predicate:: [[Target]]) typed link (parenthesized, hidden in render)
[predicate:: scalar value] typed scalar (becomes a property, not a graph edge)
`predicate` is identifier-shaped: starts with a letter, then letters /
digits / `_` / `-`. The link inside follows the bare protocol above.
## Output schema
Wikilink edges are returned as `FileEdge` (the persistence model exposed
via `file_store.get_edges(path)`). Typed scalar properties are returned
as `InlineField` — they carry no link target, just `{predicate, value}`.
## Functions
Bare extraction (back-compat — return targets only):
extract_wikilinks(text) → list[str]
extract_wikilinks_from_metadata(d) → list[str]
Structured parsing (return FileEdge):
parse_wikilinks(text) → list[FileEdge]
parse_wikilinks_from_metadata(d) → list[FileEdge]
extract_inline_fields(text) → list[InlineField]
extract_typed_edges(text) → list[FileEdge] (predicate-bearing only)
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Literal
from ..schema.file_edge import FileEdge
EdgeSource = Literal["regex", "frontmatter", "llm"]
# -- Regexes --------------------------------------------------------------
# Bare wikilink. Group(1) = target, kept as the FIRST capturing group so
# legacy callers using `m.group(1)` (e.g. wikilink rewriters) still work.
# The leading `(?:!)?` is non-capturing — read the embed prefix off
# `m.group(0).startswith("!")` instead.
WIKILINK_RE = re.compile(
r"""
(?:!)?
\[\[
(?P<target>[^\]\|\#\n]+?)
(?:\#(?P<anchor>[^\]\|\n]+))?
(?:\|(?P<alias>[^\]\n]+))?
\]\]
""",
re.VERBOSE,
)
# Dataview inline field: [pred:: value] or (pred:: value). Value may be a
# bare wikilink OR a scalar with no inner brackets. The open/close bracket
# pair isn't enforced by the regex — `_bracket_pair_ok` rejects mismatched
# `[..)`-style fragments.
INLINE_FIELD_RE = re.compile(
r"""
(?P<open>[\[\(])
(?P<predicate>[A-Za-z][\w\-]*)
\s*::\s*
(?P<value>
(?: !? \[\[ [^\]\|\#\n]+? (?:\#[^\]\|\n]+)? (?:\|[^\]\n]+)? \]\] )
|
(?: [^\]\)\n]+? )
)
(?P<close>[\]\)])
""",
re.VERBOSE,
)
# -- Auxiliary schema (typed scalar) --------------------------------------
@dataclass(frozen=True, slots=True)
class InlineField:
"""Typed scalar property — `[predicate:: value]` where value is NOT a wikilink.
Distinct from `FileEdge`: a scalar property doesn't point at another file,
so it's not a graph edge — it's an attribute on the source file.
"""
predicate: str
value: str
# -- Internal helpers -----------------------------------------------------
def _edge_from_match(
m: re.Match,
*,
predicate: str | None = None,
source: EdgeSource = "regex",
) -> FileEdge:
anchor = m.group("anchor")
alias = m.group("alias")
return FileEdge(
target=m.group("target").strip(),
anchor=anchor.strip() if anchor else None,
alias=alias.strip() if alias else None,
embed=m.group(0).startswith("!"),
predicate=predicate,
source=source,
)
def _bracket_pair_ok(open_ch: str, close_ch: str) -> bool:
return (open_ch == "[" and close_ch == "]") or (open_ch == "(" and close_ch == ")")
# -- Public API: bare extraction (back-compat) ----------------------------
def extract_wikilinks(text: str) -> list[str]:
"""Targets-only list of wikilinks found in text (no dedup).
Includes wikilinks inside typed wrappers — the watcher's graph
projection should see every link target, with or without a predicate.
Use `parse_wikilinks` if you need the predicate too.
"""
if not text:
return []
return [m.group("target").strip() for m in WIKILINK_RE.finditer(text)]
def extract_wikilinks_from_metadata(metadata: dict) -> list[str]:
"""Recursively walk a frontmatter dict, extracting wikilink targets from string leaves."""
links: list[str] = []
def _walk(value) -> None:
if isinstance(value, str):
links.extend(extract_wikilinks(value))
elif isinstance(value, dict):
for v in value.values():
_walk(v)
elif isinstance(value, (list, tuple, set)):
for item in value:
_walk(item)
_walk(metadata)
return links
# -- Public API: structured parsing ---------------------------------------
def parse_wikilinks(text: str) -> list[FileEdge]:
"""Structured parse — bare wikilinks AND typed-wrapper predicates.
Order is by source position. A wikilink inside a typed wrapper
appears once, with its predicate attached. All edges have
`source="regex"`.
"""
if not text:
return []
typed_spans: list[tuple[int, int]] = []
items: list[tuple[int, FileEdge]] = []
for m in INLINE_FIELD_RE.finditer(text):
if not _bracket_pair_ok(m.group("open"), m.group("close")):
continue
value = m.group("value").strip()
wm = WIKILINK_RE.fullmatch(value)
if wm is None:
continue # scalar — handled by extract_inline_fields
items.append((
m.start(),
_edge_from_match(wm, predicate=m.group("predicate").strip()),
))
typed_spans.append(m.span())
for m in WIKILINK_RE.finditer(text):
s, e = m.span()
if any(ts <= s and e <= te for ts, te in typed_spans):
continue # this bare match is the inner link of a typed wrapper
items.append((s, _edge_from_match(m)))
items.sort(key=lambda pair: pair[0])
return [w for _, w in items]
def extract_inline_fields(text: str) -> list[InlineField]:
"""Typed scalar fields — `[pred:: value]` whose value is NOT a wikilink.
Link-valued typed fields are returned by `parse_wikilinks` instead,
since they're graph edges rather than scalar properties.
"""
if not text:
return []
out: list[InlineField] = []
for m in INLINE_FIELD_RE.finditer(text):
if not _bracket_pair_ok(m.group("open"), m.group("close")):
continue
value = m.group("value").strip()
if WIKILINK_RE.fullmatch(value) is not None:
continue
out.append(InlineField(predicate=m.group("predicate").strip(), value=value))
return out
def extract_typed_edges(text: str) -> list[FileEdge]:
"""Sugar: just the predicate-bearing wikilinks (typed graph edges)."""
return [w for w in parse_wikilinks(text) if w.is_typed]
def parse_wikilinks_from_metadata(metadata: dict) -> list[FileEdge]:
"""Recursive frontmatter parse — frontmatter keys act as predicates.
`author: "[[John]]"` → `FileEdge(target="John", predicate="author", source="frontmatter")`.
`related: ["[[X]]", "[[Y]]"]` → two `FileEdge(predicate="related", ...)`.
A bare wikilink in a string value with no parent key keeps `predicate=None`.
Inline `[pred:: ...]` syntax inside a string value still wins over the
inherited frontmatter key (predicate carried from the inline parse).
All edges have `source="frontmatter"`.
"""
links: list[FileEdge] = []
def _walk(value, predicate: str | None) -> None:
if isinstance(value, str):
for w in parse_wikilinks(value):
effective = w.predicate if w.predicate is not None else predicate
links.append(FileEdge(
target=w.target,
anchor=w.anchor,
alias=w.alias,
embed=w.embed,
predicate=effective,
source="frontmatter",
))
elif isinstance(value, dict):
for k, v in value.items():
_walk(v, predicate=str(k))
elif isinstance(value, (list, tuple, set)):
for item in value:
_walk(item, predicate=predicate)
_walk(metadata, predicate=None)
return links

0
reme2/vector_based/__init__.py
View file