diff --git a/.gitignore b/.gitignore
index fa4adba4..81830405 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,4 +43,6 @@ meta_memory/*
*.db
memories/*
.reme/*
-/test_data
\ No newline at end of file
+/vault
+/reme-plugin
+/reme2/vault
\ No newline at end of file
diff --git a/reme/memory/vector_based/personal/personal_retriever.py b/reme/memory/vector_based/personal/personal_retriever.py
index 2b0c094e..b8803717 100644
--- a/reme/memory/vector_based/personal/personal_retriever.py
+++ b/reme/memory/vector_based/personal/personal_retriever.py
@@ -106,5 +106,5 @@ class PersonalRetriever(BaseMemoryAgent):
],
)
- result["retrieved_nodes"] = self.retri eved_nodes
+ result["retrieved_nodes"] = self.retrieved_nodes
return result
diff --git a/reme2/application.py b/reme2/application.py
index 89c4f667..10f6ed86 100644
--- a/reme2/application.py
+++ b/reme2/application.py
@@ -114,14 +114,14 @@ class Application(BaseComponent):
except Exception as e:
self.logger.exception(f"Failed to close component {component.__class__.__name__}: {e}")
- async def run_job(self, name: str, **kwargs) -> Response:
+ async def run_job(self, name: str, /, **kwargs) -> Response:
"""Execute a registered job by name."""
if name not in self.context.jobs:
raise KeyError(f"Job '{name}' not found")
job = self.context.jobs[name]
return await job(**kwargs)
- async def run_stream_job(self, name: str, **kwargs) -> AsyncGenerator[StreamChunk, None]:
+ async def run_stream_job(self, name: str, /, **kwargs) -> AsyncGenerator[StreamChunk, None]:
"""Execute a streaming job and yield chunks."""
if name not in self.context.jobs:
raise KeyError(f"Job '{name}' not found")
@@ -129,10 +129,10 @@ class Application(BaseComponent):
stream_queue = asyncio.Queue()
task = asyncio.create_task(job(stream_queue=stream_queue, **kwargs))
async for chunk in execute_stream_task(
- stream_queue=stream_queue,
- task=task,
- task_name=name,
- output_format="chunk",
+ stream_queue=stream_queue,
+ task=task,
+ task_name=name,
+ output_format="chunk",
):
assert isinstance(chunk, StreamChunk)
yield chunk
diff --git a/reme2/component/__init__.py b/reme2/component/__init__.py
index 6b3942ec..80e89d70 100644
--- a/reme2/component/__init__.py
+++ b/reme2/component/__init__.py
@@ -2,7 +2,9 @@
from . import as_llm
from . import as_llm_formatter
-from . import chunk_store
+from . import as_token_counter
+from . import edge_extractor
+from . import file_store
from . import client
from . import embedding
from . import file_parser
@@ -27,7 +29,9 @@ __all__ = [
# base components
"as_llm",
"as_llm_formatter",
- "chunk_store",
+ "as_token_counter",
+ "edge_extractor",
+ "file_store",
"client",
"embedding",
"file_parser",
diff --git a/reme2/component/as_llm_formatter/reme_openai_chat_formatter.py b/reme2/component/as_llm_formatter/reme_openai_chat_formatter.py
index 66dbfd51..a501da26 100644
--- a/reme2/component/as_llm_formatter/reme_openai_chat_formatter.py
+++ b/reme2/component/as_llm_formatter/reme_openai_chat_formatter.py
@@ -35,8 +35,8 @@ class ReMeOpenAIChatFormatter(OpenAIChatFormatter):
"""Extends OpenAIChatFormatter with tool result image promotion and reasoning content support."""
async def _format(
- self,
- msgs: list[Msg],
+ self,
+ msgs: list[Msg],
) -> list[dict[str, Any]]:
"""Format messages into OpenAI API format.
@@ -108,7 +108,7 @@ class ReMeOpenAIChatFormatter(OpenAIChatFormatter):
TextBlock(
type="text",
text="The following are the image contents from the tool "
- f"result of '{block['name']}':",
+ f"result of '{block['name']}':",
),
*promoted_blocks,
TextBlock(type="text", text=""),
diff --git a/reme2/component/as_token_counter/estimate_token_counter.py b/reme2/component/as_token_counter/estimate_token_counter.py
index 01f106c7..eca3f446 100644
--- a/reme2/component/as_token_counter/estimate_token_counter.py
+++ b/reme2/component/as_token_counter/estimate_token_counter.py
@@ -12,9 +12,9 @@ class EstimatedTokenCounter(TokenCounterBase):
"""
def __init__(
- self,
- estimate_divisor: float = 4,
- encoding: str = "utf-8",
+ self,
+ estimate_divisor: float = 4,
+ encoding: str = "utf-8",
):
"""Initialize the estimated token counter.
diff --git a/reme2/component/base_component.py b/reme2/component/base_component.py
index 0d5c1e9a..4cab9f58 100644
--- a/reme2/component/base_component.py
+++ b/reme2/component/base_component.py
@@ -20,11 +20,11 @@ class BaseComponent(ABC):
component_type = ComponentEnum.BASE
def __init__(
- self,
- name: str | None = None,
- backend: str = "",
- app_context: "ApplicationContext | None" = None,
- **kwargs,
+ self,
+ name: str | None = None,
+ backend: str = "",
+ app_context: "ApplicationContext | None" = None,
+ **kwargs,
) -> None:
self.name: str = name or self.__class__.__name__
self.backend: str = backend
diff --git a/reme2/component/base_step.py b/reme2/component/base_step.py
index 52cb699e..e149fa4a 100644
--- a/reme2/component/base_step.py
+++ b/reme2/component/base_step.py
@@ -8,12 +8,11 @@ from agentscope.model import ChatModelBase
from agentscope.token import TokenCounterBase
from .base_component import BaseComponent
-from .chunk_store import BaseChunkStore
+from .file_store import BaseFileStore
from .embedding import BaseEmbeddingModel
from .prompt_handler import PromptHandler
from .runtime_context import RuntimeContext
from ..enumeration import ComponentEnum
-from ..schema.file_graph import FileGraph
class BaseStep(BaseComponent):
@@ -28,12 +27,12 @@ class BaseStep(BaseComponent):
return instance
def __init__(
- self,
- language: str = "",
- prompt_dict: dict[str, str] | None = None,
- input_mapping: dict[str, str] | None = None,
- output_mapping: dict[str, str] | None = None,
- **kwargs,
+ self,
+ language: str = "",
+ prompt_dict: dict[str, str] | None = None,
+ input_mapping: dict[str, str] | None = None,
+ output_mapping: dict[str, str] | None = None,
+ **kwargs,
):
super().__init__(**kwargs)
self.language = language
@@ -66,6 +65,17 @@ class BaseStep(BaseComponent):
comp = self.app_context.components[key][name]
return getattr(comp, attr) if attr else comp
+ def _get_component_optional(self, key: ComponentEnum, name: str = "default", attr: str | None = None):
+ """Like `_get_component` but returns None instead of raising when
+ the component / attribute is missing. For features that should
+ gracefully degrade when an LLM (etc.) isn't configured."""
+ if self.app_context is None:
+ return None
+ comp = self.app_context.components.get(key, {}).get(name)
+ if comp is None:
+ return None
+ return getattr(comp, attr, None) if attr else comp
+
@property
def as_llm(self) -> ChatModelBase:
name = self.kwargs.get("as_llm", "default")
@@ -74,31 +84,45 @@ class BaseStep(BaseComponent):
@property
def as_llm_formatter(self) -> FormatterBase:
name = self.kwargs.get("as_llm_formatter", "default")
- return name if isinstance(name, FormatterBase) else self._get_component(ComponentEnum.AS_LLM_FORMATTER, name,
- "formatter")
+ return (
+ name
+ if isinstance(name, FormatterBase)
+ else self._get_component(
+ ComponentEnum.AS_LLM_FORMATTER,
+ name,
+ "formatter",
+ )
+ )
@property
def as_token_counter(self) -> TokenCounterBase:
name = self.kwargs.get("as_token_counter", "default")
- return name if isinstance(name, TokenCounterBase) else self._get_component(ComponentEnum.AS_TOKEN_COUNTER, name,
- "token_counter")
+ return (
+ name
+ if isinstance(name, TokenCounterBase)
+ else self._get_component(
+ ComponentEnum.AS_TOKEN_COUNTER,
+ name,
+ "token_counter",
+ )
+ )
@property
- def chunk_store(self) -> BaseChunkStore:
- name = self.kwargs.get("chunk_store", "default")
- return name if isinstance(name, BaseChunkStore) else self._get_component(ComponentEnum.CHUNK_STORE, name)
-
- @property
- def file_graph(self) -> FileGraph:
- name = self.kwargs.get("file_watcher", "default")
- watcher = self._get_component(ComponentEnum.FILE_WATCHER, name)
- return watcher.file_graph
+ def file_store(self) -> BaseFileStore:
+ name = self.kwargs.get("file_store", "default")
+ return name if isinstance(name, BaseFileStore) else self._get_component(ComponentEnum.FILE_STORE, name)
@property
def embedding(self) -> BaseEmbeddingModel:
name = self.kwargs.get("embedding", "default")
- return name if isinstance(name, BaseEmbeddingModel) else self._get_component(ComponentEnum.EMBEDDING_MODEL,
- name)
+ return (
+ name
+ if isinstance(name, BaseEmbeddingModel)
+ else self._get_component(
+ ComponentEnum.EMBEDDING_MODEL,
+ name,
+ )
+ )
def prompt_format(self, prompt_name: str, **kwargs) -> str:
return self.prompt.prompt_format(prompt_name=prompt_name, **kwargs)
diff --git a/reme2/component/chunk_store/__init__.py b/reme2/component/chunk_store/__init__.py
deleted file mode 100644
index 0eb1470f..00000000
--- a/reme2/component/chunk_store/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""Chunk store module.
-
-Storage backends for FileChunks with vector and full-text search.
-File metadata is managed by FileGraph, not by ChunkStore.
-"""
-
-from .base_chunk_store import BaseChunkStore
-from .chroma_chunk_store import ChromaChunkStore
-from .local_chunk_store import LocalChunkStore
-from .sqlite_chunk_store import SqliteChunkStore
-
-__all__ = [
- "BaseChunkStore",
- "ChromaChunkStore",
- "LocalChunkStore",
- "SqliteChunkStore",
-]
diff --git a/reme2/component/chunk_store/base_chunk_store.py b/reme2/component/chunk_store/base_chunk_store.py
deleted file mode 100644
index 834584da..00000000
--- a/reme2/component/chunk_store/base_chunk_store.py
+++ /dev/null
@@ -1,241 +0,0 @@
-"""Abstract base class for chunk storage backends."""
-
-import re
-from abc import abstractmethod
-from pathlib import Path
-
-from ..base_component import BaseComponent
-from ..embedding import BaseEmbeddingModel
-from ...enumeration import ComponentEnum
-from ...schema import ChunkFilter, FileChunk
-
-
-class BaseChunkStore(BaseComponent):
- """Abstract base class for chunk storage backends.
-
- Handles chunk persistence and retrieval (vector / keyword / hybrid search).
- File-level metadata and search filter resolution live in FileGraph; this
- layer only consumes a compiled ChunkFilter (path set) for restricting search.
- """
-
- component_type = ComponentEnum.CHUNK_STORE
-
- def __init__(
- self,
- store_name: str,
- db_path: str | Path,
- embedding_model: str = "default",
- fts_enabled: bool = True,
- **kwargs,
- ):
- super().__init__(**kwargs)
- self._embedding_model_name: str = embedding_model
- self.embedding_model: BaseEmbeddingModel | None = None
- self.store_name: str = store_name
- self.db_path: Path = Path(db_path)
- self.db_path.mkdir(parents=True, exist_ok=True)
- self.vector_enabled: bool = bool(embedding_model)
- self.fts_enabled: bool = fts_enabled
-
- if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
- raise ValueError(
- f"Invalid store name '{store_name}'. Only alphanumeric characters and underscores are allowed.",
- )
- if not self.vector_enabled and not self.fts_enabled:
- raise ValueError("At least one of embedding_model or fts_enabled must be set.")
-
- async def _start(self):
- if not self._embedding_model_name:
- return
- assert self.app_context is not None, "app_context must be provided"
- models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
- if self._embedding_model_name not in models:
- raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
- model = models[self._embedding_model_name]
- if not isinstance(model, BaseEmbeddingModel):
- raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
- self.embedding_model = model
-
- async def _close(self):
- """Release embedding model reference."""
- self.embedding_model = None
-
- @property
- def embedding_dim(self) -> int:
- """Return the embedding dimensionality (default 1024)."""
- return self.embedding_model.dimensions if self.embedding_model else 1024
-
- def _disable_vector_search(self, reason: str = "embedding API error") -> None:
- """Disable vector search and log a warning."""
- if self.vector_enabled:
- self.logger.warning(f"[{self.store_name}] Disabling vector search: {reason}")
- self.vector_enabled = False
-
- async def _get_embeddings_safe(self, texts: list[str], **kwargs) -> list[list[float]] | None:
- """Get embeddings, returning None if vector search is disabled or an error occurs."""
- if not self.vector_enabled:
- return None
- try:
- assert self.embedding_model is not None, "Embedding model not initialized"
- return await self.embedding_model.get_embeddings(texts, **kwargs)
- except Exception as e:
- self._disable_vector_search(str(e))
- return None
-
- async def get_embedding(self, query: str, **kwargs) -> list[float] | None:
- """Get embedding for a single query string."""
- result = await self._get_embeddings_safe([query], **kwargs)
- return result[0] if result else None
-
- async def get_embeddings(self, queries: list[str], **kwargs) -> list[list[float]] | None:
- """Get embeddings for a batch of query strings."""
- return await self._get_embeddings_safe(queries, **kwargs)
-
- async def get_chunk_embedding(self, chunk: FileChunk, **kwargs) -> FileChunk:
- """Attach embedding to a single FileChunk."""
- chunk.embedding = await self.get_embedding(chunk.text, **kwargs)
- return chunk
-
- async def get_chunk_embeddings(self, chunks: list[FileChunk], **kwargs) -> list[FileChunk]:
- """Attach embeddings to a batch of FileChunk."""
- if not chunks:
- return chunks
- embeddings = await self.get_embeddings([c.text for c in chunks], **kwargs)
- if embeddings and len(embeddings) == len(chunks):
- for chunk, emb in zip(chunks, embeddings):
- chunk.embedding = emb
- else:
- for chunk in chunks:
- chunk.embedding = None
- return chunks
-
- # -- Keyword scoring utility --------------------------------------------
-
- @staticmethod
- def _score_keyword_match(query: str, text: str) -> float:
- """Score a keyword match using word-match ratio + phrase bonus."""
- words = query.split()
- if not words:
- return 0.0
- query_lower = query.lower()
- words_lower = [w.lower() for w in words]
- text_lower = text.lower()
- n_words = len(words)
-
- match_count = sum(1 for w in words_lower if w in text_lower)
- if match_count == 0:
- return 0.0
-
- base_score = match_count / n_words
- phrase_bonus = 0.2 if n_words > 1 and query_lower in text_lower else 0.0
- return min(1.0, base_score + phrase_bonus)
-
- # -- Hybrid search (concrete, delegates to abstract vector/keyword) -----
-
- async def hybrid_search(
- self,
- query: str,
- limit: int,
- vector_weight: float = 0.7,
- candidate_multiplier: float = 3.0,
- chunk_filter: ChunkFilter | None = None,
- ) -> list[FileChunk]:
- """Perform hybrid search combining vector and keyword results."""
- assert 0.0 <= vector_weight <= 1.0
-
- candidates = min(200, max(1, int(limit * candidate_multiplier)))
- text_weight = 1.0 - vector_weight
-
- if self.vector_enabled and self.fts_enabled:
- keyword_results = await self.keyword_search(query, candidates, chunk_filter)
- vector_results = await self.vector_search(query, candidates, chunk_filter)
-
- if not keyword_results:
- return vector_results[:limit]
- if not vector_results:
- return keyword_results[:limit]
-
- merged = self._merge_hybrid_results(
- vector_results,
- keyword_results,
- vector_weight,
- text_weight,
- )
- return merged[:limit]
- elif self.vector_enabled:
- return await self.vector_search(query, limit, chunk_filter)
- elif self.fts_enabled:
- return await self.keyword_search(query, limit, chunk_filter)
- return []
-
- @staticmethod
- def _merge_hybrid_results(
- vector: list[FileChunk],
- keyword: list[FileChunk],
- vector_weight: float,
- text_weight: float,
- ) -> list[FileChunk]:
- """Merge vector and keyword results with weighted scoring."""
- merged: dict[str, FileChunk] = {}
-
- for result in vector:
- v_score = result.scores.get("vector", 0)
- result.scores["score"] = v_score * vector_weight
- merged[result.unique_key] = result
-
- for result in keyword:
- key = result.unique_key
- k_score = result.scores.get("keyword", 0)
- if key in merged:
- merged[key].scores["score"] += k_score * text_weight
- else:
- result.scores["score"] = k_score * text_weight
- merged[key] = result
-
- results = list(merged.values())
- results.sort(key=lambda r: r.score, reverse=True)
- return results
-
- # -- Filter utility -----------------------------------------------------
-
- @staticmethod
- def _apply_filter(chunks: list[FileChunk], chunk_filter: ChunkFilter | None) -> list[FileChunk]:
- if chunk_filter is None or chunk_filter.resolved_paths is None:
- return chunks
- return [c for c in chunks if chunk_filter.match_path(c.path)]
-
- # -- Abstract methods ---------------------------------------------------
-
- @abstractmethod
- async def clear_all(self):
- """Clear all indexed data."""
-
- @abstractmethod
- async def upsert_chunks(self, path: str, chunks: list[FileChunk]):
- """Insert or update all chunks for a file path."""
-
- @abstractmethod
- async def delete_chunks(self, path: str):
- """Delete all chunks for a file path."""
-
- @abstractmethod
- async def get_chunks(self, path: str) -> list[FileChunk]:
- """Get all chunks for a file path."""
-
- @abstractmethod
- async def vector_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
- ) -> list[FileChunk]:
- """Perform vector similarity search, optionally restricted by chunk_filter."""
-
- @abstractmethod
- async def keyword_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
- ) -> list[FileChunk]:
- """Perform full-text/keyword search, optionally restricted by chunk_filter."""
diff --git a/reme2/component/chunk_store/chroma_chunk_store.py b/reme2/component/chunk_store/chroma_chunk_store.py
deleted file mode 100644
index 44c17966..00000000
--- a/reme2/component/chunk_store/chroma_chunk_store.py
+++ /dev/null
@@ -1,226 +0,0 @@
-"""ChromaDB chunk storage backend."""
-
-import time
-
-from .base_chunk_store import BaseChunkStore
-from ..component_registry import R
-from ...schema import ChunkFilter, FileChunk
-
-try:
- import chromadb
- from chromadb.config import Settings
-
- _CHROMADB_IMPORT_ERROR: Exception | None = None
-except Exception as e:
- _CHROMADB_IMPORT_ERROR = e
- chromadb = None
- Settings = None
-
-
-@R.register("chroma")
-class ChromaChunkStore(BaseChunkStore):
- """ChromaDB chunk storage with vector and full-text search.
-
- Uses ChromaDB's native vector search and `where_document` $contains
- for keyword matching.
- """
-
- def __init__(self, **kwargs):
- if _CHROMADB_IMPORT_ERROR is not None:
- raise _CHROMADB_IMPORT_ERROR
- super().__init__(**kwargs)
- self.client: "chromadb.ClientAPI | None" = None
- self.chunks_collection: "chromadb.Collection | None" = None
-
- @property
- def collection_name(self) -> str:
- return f"chunks_{self.store_name}"
-
- # -- Lifecycle ----------------------------------------------------------
-
- async def _start(self, app_context=None) -> None:
- self.client = chromadb.PersistentClient(
- path=str(self.db_path),
- settings=Settings(allow_reset=True, anonymized_telemetry=False),
- )
- self.chunks_collection = self.client.get_or_create_collection(
- name=self.collection_name,
- metadata={"hnsw:space": "cosine"},
- )
- self.logger.info(f"ChromaChunkStore '{self.store_name}' ready: collection={self.collection_name}")
- await super()._start(app_context)
-
- async def _close(self) -> None:
- self.client = None
- self.chunks_collection = None
- await super()._close()
-
- # -- Filter helper ------------------------------------------------------
-
- @staticmethod
- def _path_where(chunk_filter: ChunkFilter | None) -> dict | None:
- if chunk_filter is None or chunk_filter.resolved_paths is None:
- return None
- paths = chunk_filter.resolved_paths
- if not paths:
- return {"path": "__nonexistent__"}
- if len(paths) == 1:
- return {"path": next(iter(paths))}
- return {"path": {"$in": list(paths)}}
-
- # -- Write operations ---------------------------------------------------
-
- async def upsert_chunks(self, path: str, chunks: list[FileChunk]) -> None:
- await self.delete_chunks(path)
-
- if not chunks:
- return
-
- chunks = await self.get_chunk_embeddings(chunks)
-
- ids, documents, embeddings, metadatas = [], [], [], []
- now = int(time.time() * 1000)
- for chunk in chunks:
- ids.append(chunk.id)
- documents.append(chunk.text)
- embeddings.append(chunk.embedding if chunk.embedding else [0.0] * self.embedding_dim)
- metadatas.append({
- "path": path,
- "start_line": chunk.start_line,
- "end_line": chunk.end_line,
- "hash": chunk.hash,
- "updated_at": now,
- })
-
- self.chunks_collection.upsert(
- ids=ids,
- documents=documents,
- embeddings=embeddings,
- metadatas=metadatas,
- )
-
- async def delete_chunks(self, path: str) -> None:
- results = self.chunks_collection.get(where={"path": path}, include=[])
- if results["ids"]:
- self.chunks_collection.delete(ids=results["ids"])
-
- # -- Read operations ----------------------------------------------------
-
- async def get_chunks(self, path: str) -> list[FileChunk]:
- results = self.chunks_collection.get(where={"path": path}, include=["documents", "metadatas"])
- chunks: list[FileChunk] = []
- for cid, md, text in zip(results["ids"], results["metadatas"], results["documents"]):
- chunks.append(self._chunk_from_chroma(cid, md, text))
- chunks.sort(key=lambda c: c.start_line)
- return chunks
-
- # -- Search helpers -----------------------------------------------------
-
- @staticmethod
- def _chunk_from_chroma(chunk_id: str, md: dict, text: str, embedding=None) -> FileChunk:
- return FileChunk(
- id=chunk_id,
- path=md["path"],
- start_line=md["start_line"],
- end_line=md["end_line"],
- text=text,
- hash=md["hash"],
- embedding=embedding,
- )
-
- # -- Search operations --------------------------------------------------
-
- async def vector_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
- ) -> list[FileChunk]:
- if not self.vector_enabled or not query:
- return []
-
- query_embedding = await self.get_embedding(query)
- if not query_embedding:
- return []
-
- try:
- results = self.chunks_collection.query(
- query_embeddings=[query_embedding],
- n_results=limit,
- where=self._path_where(chunk_filter),
- include=["documents", "metadatas", "distances"],
- )
- except Exception as e:
- self.logger.error(f"Vector search failed: {e}")
- return []
-
- chunks = []
- if results["ids"] and results["ids"][0]:
- for i, cid in enumerate(results["ids"][0]):
- md = results["metadatas"][0][i]
- distance = results["distances"][0][i]
- score = max(0.0, 1.0 - distance / 2.0)
-
- chunk = self._chunk_from_chroma(cid, md, results["documents"][0][i])
- chunk.scores = {"vector": score, "score": score}
- chunks.append(chunk)
-
- chunks.sort(key=lambda c: c.score, reverse=True)
- return chunks[:limit]
-
- async def keyword_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
- ) -> list[FileChunk]:
- if not self.fts_enabled or not query:
- return []
-
- words = query.split()
- if not words:
- return []
-
- word_variants = set()
- for word in words:
- word_variants.add(word)
- word_variants.add(word.lower())
- word_variants.add(word.capitalize())
- word_variants.add(word.upper())
- variants_list = list(word_variants)
-
- if len(variants_list) == 1:
- where_document: dict = {"$contains": variants_list[0]}
- else:
- where_document = {"$or": [{"$contains": w} for w in variants_list]}
-
- results = self.chunks_collection.get(
- where=self._path_where(chunk_filter),
- where_document=where_document,
- include=["documents", "metadatas"],
- )
-
- chunks = []
- for i, cid in enumerate(results["ids"]):
- md = results["metadatas"][i]
- text = results["documents"][i]
- score = self._score_keyword_match(query, text)
- if score == 0.0:
- continue
-
- chunk = self._chunk_from_chroma(cid, md, text)
- chunk.scores = {"keyword": score, "score": score}
- chunks.append(chunk)
-
- chunks.sort(key=lambda c: c.score, reverse=True)
- return chunks[:limit]
-
- # -- Clear --------------------------------------------------------------
-
- async def clear_all(self) -> None:
- self.client.delete_collection(name=self.collection_name)
- self.chunks_collection = self.client.get_or_create_collection(
- name=self.collection_name,
- metadata={"hnsw:space": "cosine"},
- )
- self.logger.info(f"Cleared all data from ChromaChunkStore '{self.store_name}'")
diff --git a/reme2/component/client/http_client.py b/reme2/component/client/http_client.py
index c562bf11..0ac423ab 100644
--- a/reme2/component/client/http_client.py
+++ b/reme2/component/client/http_client.py
@@ -15,12 +15,12 @@ class HttpClient(BaseClient):
"""HTTP client for ReMe service."""
def __init__(
- self,
- action: str,
- host: str | None = None,
- port: int | None = None,
- timeout: float = 30.0,
- **kwargs,
+ self,
+ action: str,
+ host: str | None = None,
+ port: int | None = None,
+ timeout: float = 30.0,
+ **kwargs,
):
super().__init__(**kwargs)
diff --git a/reme2/component/component_registry.py b/reme2/component/component_registry.py
index 16ead1e0..f8cb85e6 100644
--- a/reme2/component/component_registry.py
+++ b/reme2/component/component_registry.py
@@ -38,9 +38,9 @@ class ComponentRegistry:
return cls
def register(
- self,
- cls_or_name: type[T] | str,
- name: str | None = None,
+ self,
+ cls_or_name: type[T] | str,
+ name: str | None = None,
) -> Callable[[type[T]], type[T]] | type[T]:
"""Register a component class. Supports direct and decorator modes."""
# Direct registration: R.register(MyClass, "name")
diff --git a/reme2/component/edge_extractor/__init__.py b/reme2/component/edge_extractor/__init__.py
new file mode 100644
index 00000000..021746c8
--- /dev/null
+++ b/reme2/component/edge_extractor/__init__.py
@@ -0,0 +1,11 @@
+"""Typed-edge extractors — `ComponentEnum.EDGE_EXTRACTOR`."""
+
+from .base_edge_extractor import BaseEdgeExtractor
+from .llm_edge_extractor import LLMEdgeExtractor
+from .regex_edge_extractor import RegexEdgeExtractor
+
+__all__ = [
+ "BaseEdgeExtractor",
+ "RegexEdgeExtractor",
+ "LLMEdgeExtractor",
+]
diff --git a/reme2/component/edge_extractor/base_edge_extractor.py b/reme2/component/edge_extractor/base_edge_extractor.py
new file mode 100644
index 00000000..110021bc
--- /dev/null
+++ b/reme2/component/edge_extractor/base_edge_extractor.py
@@ -0,0 +1,49 @@
+"""Abstract base class for typed-edge extractors.
+
+A `BaseEdgeExtractor` ingests a markdown file's body text + frontmatter
+dict and returns a flat list of `FileEdge` objects. The concrete
+implementations realize the two routes described in `structure.md`
+§"图谱关系的双态抽取":
+
+ Fast route (regex) — RegexEdgeExtractor
+ Slow route (LLM IE) — LLMEdgeExtractor
+
+Edges are returned with the `source` field already populated so
+downstream consumers (file_store indexing, retrievers, maintainer) can
+reason about provenance.
+
+Resolution to absolute paths and dedup against already-indexed edges
+are NOT this layer's responsibility — `target` stays raw, mirroring
+how it appears in source.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import FileEdge
+
+
+class BaseEdgeExtractor(BaseComponent):
+ """Pluggable typed-edge extractor (`ComponentEnum.EDGE_EXTRACTOR`)."""
+
+ component_type = ComponentEnum.EDGE_EXTRACTOR
+
+ @abstractmethod
+ async def extract(
+ self,
+ text: str,
+ metadata: dict | None = None,
+ path: str | None = None,
+ ) -> list[FileEdge]:
+ """Return the edges discovered in `text` and `metadata`.
+
+ Args:
+ text: The body text of the source file (frontmatter stripped).
+ metadata: Parsed YAML frontmatter dict, or None.
+ path: Optional absolute path of the source file. Slow-route
+ extractors use this for logging / source-text caching;
+ fast-route ignores it.
+ """
diff --git a/reme2/component/edge_extractor/llm_edge_extractor.py b/reme2/component/edge_extractor/llm_edge_extractor.py
new file mode 100644
index 00000000..30e1e4d3
--- /dev/null
+++ b/reme2/component/edge_extractor/llm_edge_extractor.py
@@ -0,0 +1,391 @@
+"""LLMEdgeExtractor — slow route per `structure.md` §"双态抽取".
+
+ReAct agent over the memory store: instead of a single-shot extraction
+from the file's text alone, the agent browses the vault via read-only
+memory tools (`memory_get` / `memory_list` / `memory_links` /
+`memory_backlinks` / `memory_resolve_wikilink`) to ground its triples
+in entities that already exist. It finalizes by calling `emit_edges`
+with the JSON triples it wants to commit.
+
+Each accepted triple becomes a `FileEdge` with `source="llm"` and
+`confidence` populated from the model's self-rating.
+
+The extractor does NOT write back to the source file — that's the
+caller's job (typically the parser pipeline). It also does NOT
+entity-resolve targets; the raw `object` string from the model lands
+in `FileEdge.target` and is matched at the file_store layer via the
+usual wikilink resolution pipeline.
+
+Inputs that exceed `max_input_chars` are truncated; long files
+should be chunked by the caller before invoking this extractor.
+
+Triples are filtered by `min_confidence` and predicate-normalized
+(lowercase, non-identifier characters → underscore) when
+`predicate_normalizer=True`.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import frontmatter
+from agentscope.agent import ReActAgent
+from agentscope.formatter import FormatterBase
+from agentscope.message import Msg, TextBlock
+from agentscope.model import ChatModelBase
+from agentscope.tool import Toolkit, ToolResponse
+from pydantic import BaseModel, Field
+
+from .base_edge_extractor import BaseEdgeExtractor
+from ..component_registry import R
+from ..file_store import BaseFileStore
+from ...enumeration import ComponentEnum
+from ...schema import FileEdge
+
+
+_SYSTEM_PROMPT = """You are a high-precision information-extraction agent for a personal markdown vault.
+
+Your job for one file: identify subject-predicate-object triples that capture meaningful, lasting relations between named entities, then emit them by calling `emit_edges`. Quality > recall.
+
+You have read-only access to the vault via these tools:
+ - `memory_list(path_prefix, tags, limit)` — browse indexed files
+ - `memory_get(path)` — read frontmatter + body + edges of one file
+ - `memory_resolve_wikilink(wikilink)` — check whether `[[X]]` resolves to a real path
+ - `memory_links(path)` — outgoing edges from `path`
+ - `memory_backlinks(path)` — incoming edges to `path`
+ - `emit_edges(triples)` — finalize: list of {subject, predicate, object, confidence}
+
+Recommended loop (a few iterations is fine; do NOT exhaust the vault):
+ 1. Read the source file's text and metadata (provided in the user message).
+ 2. Generate candidate object entities from the text.
+ 3. Probe the vault: prefer `memory_resolve_wikilink` to confirm a candidate exists; fall back to `memory_list` for fuzzy browsing only when needed.
+ 4. Drop candidates that are pronouns / generic concepts / vague references / self-references.
+ 5. Call `emit_edges` ONCE with the final filtered set, then stop.
+
+Triple rules:
+ - subject: the source file's main entity (typically the file name / first H1).
+ - predicate: identifier-shaped, lowercase, words joined with underscore (e.g. `works_at`, `authored`, `depends_on`, `mentions`).
+ - object: the target entity name as it appears in the text — do NOT wrap with `[[ ]]`.
+ - confidence: float in [0, 1]; the parser drops anything below `min_confidence`.
+
+Skip silently: pronouns, self-referential relations, transient events, generic concepts.
+If nothing meaningful is present, call `emit_edges([])` and stop."""
+
+
+_USER_TEMPLATE = """Source file: {path}
+
+Frontmatter:
+{frontmatter}
+
+Body:
+{text}
+
+Extract triples grounded in the vault now. Use the read tools as needed, then call `emit_edges` ONCE with the final list."""
+
+
+class _Triple(BaseModel):
+ """One subject-predicate-object triple from the LLM."""
+
+ subject: str = Field(description="Source entity of the relation.")
+ predicate: str = Field(description="Identifier-shaped relation name.")
+ object: str = Field(description="Target entity name.")
+ confidence: float = Field(ge=0.0, le=1.0, description="Self-rated confidence.")
+
+
+_LINK_WRAPPER_RE = re.compile(r"^!?\[\[([^\]\|\#]+?)(?:[#\|][^\]]*)?\]\]$")
+_PREDICATE_NORM_RE = re.compile(r"[^a-z0-9_]+")
+
+
+def _strip_link_wrapper(s: str) -> str:
+ """Strip `[[X]]` / `![[X#h|alias]]` wrappers if the LLM emitted them."""
+ s = s.strip().strip('"\'')
+ if m := _LINK_WRAPPER_RE.match(s):
+ return m.group(1).strip()
+ return s
+
+
+def _normalize_predicate(pred: str) -> str:
+ """Lowercase + collapse non-identifier chars to `_`."""
+ pred = pred.strip().lower()
+ pred = _PREDICATE_NORM_RE.sub("_", pred).strip("_")
+ return pred or "related"
+
+
+def _text_response(payload: object) -> ToolResponse:
+ text = json.dumps(payload, ensure_ascii=False, indent=2, default=str)
+ return ToolResponse(content=[TextBlock(type="text", text=text)])
+
+
+@R.register("llm")
+class LLMEdgeExtractor(BaseEdgeExtractor):
+ """LLM-driven typed-edge extractor (slow route, ReAct over memory tools)."""
+
+ def __init__(
+ self,
+ as_llm: str = "default",
+ as_llm_formatter: str = "default",
+ file_store: str = "default",
+ max_input_chars: int = 8000,
+ min_confidence: float = 0.5,
+ predicate_normalizer: bool = True,
+ max_iters: int = 8,
+ console_enabled: bool = False,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self._as_llm_name: str = as_llm
+ self._as_llm_formatter_name: str = as_llm_formatter
+ self._file_store_name: str = file_store
+ self.max_input_chars: int = int(max_input_chars)
+ self.min_confidence: float = float(min_confidence)
+ self.predicate_normalizer: bool = bool(predicate_normalizer)
+ self.max_iters: int = int(max_iters)
+ self.console_enabled: bool = bool(console_enabled)
+ self.as_llm: ChatModelBase | None = None
+ self.as_llm_formatter: FormatterBase | None = None
+ self.file_store: BaseFileStore | None = None
+
+ async def _start(self) -> None:
+ assert self.app_context is not None, "LLMEdgeExtractor requires app_context"
+
+ llms = self.app_context.components.get(ComponentEnum.AS_LLM, {})
+ wrapper = llms.get(self._as_llm_name)
+ if wrapper is None:
+ raise ValueError(f"as_llm '{self._as_llm_name}' not configured")
+ model = getattr(wrapper, "model", None)
+ if not isinstance(model, ChatModelBase):
+ raise TypeError(
+ f"as_llm '{self._as_llm_name}'.model is {type(model).__name__}, "
+ f"expected ChatModelBase",
+ )
+ self.as_llm = model
+
+ formatters = self.app_context.components.get(ComponentEnum.AS_LLM_FORMATTER, {})
+ fwrapper = formatters.get(self._as_llm_formatter_name)
+ formatter = getattr(fwrapper, "formatter", None) if fwrapper is not None else None
+ if not isinstance(formatter, FormatterBase):
+ raise TypeError(
+ f"as_llm_formatter '{self._as_llm_formatter_name}' missing or wrong type",
+ )
+ self.as_llm_formatter = formatter
+
+ stores = self.app_context.components.get(ComponentEnum.FILE_STORE, {})
+ store = stores.get(self._file_store_name)
+ if not isinstance(store, BaseFileStore):
+ raise ValueError(
+ f"file_store '{self._file_store_name}' not configured for LLMEdgeExtractor",
+ )
+ self.file_store = store
+
+ async def _close(self) -> None:
+ self.as_llm = None
+ self.as_llm_formatter = None
+ self.file_store = None
+
+ async def extract(
+ self,
+ text: str,
+ metadata: dict | None = None,
+ path: str | None = None,
+ ) -> list[FileEdge]:
+ if not text or not text.strip():
+ return []
+ if self.as_llm is None or self.as_llm_formatter is None or self.file_store is None:
+ raise RuntimeError("LLMEdgeExtractor not started; call .start() first")
+
+ body = text if len(text) <= self.max_input_chars else text[: self.max_input_chars]
+
+ toolkit, sink = self._build_toolkit()
+
+ agent = ReActAgent(
+ name="llm_edge_extractor",
+ model=self.as_llm,
+ sys_prompt=_SYSTEM_PROMPT,
+ formatter=self.as_llm_formatter,
+ toolkit=toolkit,
+ max_iters=self.max_iters,
+ )
+ agent.set_console_output_enabled(self.console_enabled)
+
+ user_msg = _USER_TEMPLATE.format(
+ path=path or "(unknown)",
+ frontmatter=json.dumps(metadata or {}, ensure_ascii=False, indent=2, default=str),
+ text=body,
+ )
+
+ try:
+ await agent.reply(Msg(name="user", role="user", content=user_msg))
+ except Exception as e:
+ self.logger.warning(f"LLM IE agent failed for {path}: {e}")
+ return []
+
+ return self._sink_to_edges(sink)
+
+ # -- Tool plumbing ------------------------------------------------------
+
+ def _build_toolkit(self) -> tuple[Toolkit, list[_Triple]]:
+ """Build a read-only memory toolkit + an `emit_edges` sink.
+
+ Returns `(toolkit, sink)`. The sink is the mutable list the
+ `emit_edges` tool appends parsed triples into.
+ """
+ assert self.file_store is not None
+ store = self.file_store
+ vault_root = store.vault_root or Path.cwd().resolve()
+
+ sink: list[_Triple] = []
+
+ def _resolve(p: str) -> Path:
+ pp = Path(p)
+ if not pp.is_absolute():
+ pp = vault_root / pp
+ return pp.resolve()
+
+ async def memory_get(path: str) -> ToolResponse:
+ """Read one vault file: frontmatter + body + outgoing edges.
+
+ Args:
+ path (str): Absolute or vault-relative path to the file.
+ """
+ target = _resolve(path)
+ out: dict = {"path": str(target), "exists": False}
+ meta = store.get_file_meta(str(target))
+ if meta is not None:
+ edges = store.get_edges(str(target))
+ out.update({
+ "exists": True,
+ "metadata": meta.metadata,
+ "edges": [e.model_dump(exclude_none=True) for e in edges],
+ })
+ if target.is_file():
+ try:
+ raw = target.read_text(encoding="utf-8")
+ post = frontmatter.loads(raw)
+ out["exists"] = True
+ out["metadata"] = dict(post.metadata)
+ out["content"] = post.content
+ except Exception as e:
+ out["read_error"] = str(e)
+ return _text_response(out)
+
+ async def memory_list(
+ path_prefix: str = "",
+ tags: list[str] | None = None,
+ limit: int = 50,
+ ) -> ToolResponse:
+ """List indexed vault files, filtered by path prefix and tags.
+
+ Args:
+ path_prefix (str): Restrict to paths starting with this prefix.
+ tags (list[str] | None): All tags must be present on a file.
+ limit (int): Cap on returned items.
+ """
+ items: list[dict] = []
+ for p, meta in store.nodes.items():
+ if path_prefix and not p.startswith(path_prefix):
+ continue
+ md = meta.metadata or {}
+ if tags:
+ file_tags = set(md.get("tags", []) or [])
+ if not all(t in file_tags for t in tags):
+ continue
+ items.append({"path": p, "file": meta.file, "metadata": md})
+ if len(items) >= limit:
+ break
+ return _text_response({"items": items, "count": len(items)})
+
+ async def memory_resolve_wikilink(wikilink: str) -> ToolResponse:
+ """Resolve a `[[wikilink]]` to its absolute path (or null if dangling).
+
+ Args:
+ wikilink (str): The bare wikilink target (no brackets).
+ """
+ hit = store.resolve_wikilink(wikilink)
+ return _text_response({"wikilink": wikilink, "path": hit})
+
+ async def memory_links(path: str) -> ToolResponse:
+ """Outgoing edges from `path` (resolved targets only).
+
+ Args:
+ path (str): Absolute or vault-relative path.
+ """
+ target = _resolve(path)
+ out = [
+ {"path": m.path, "predicate": e.predicate, "metadata": m.metadata}
+ for m, e in store.get_links(str(target))
+ ]
+ return _text_response({"path": str(target), "links": out})
+
+ async def memory_backlinks(path: str) -> ToolResponse:
+ """Incoming edges to `path` (files that link TO it).
+
+ Args:
+ path (str): Absolute or vault-relative path.
+ """
+ target = _resolve(path)
+ out = [
+ {"path": m.path, "predicate": e.predicate, "metadata": m.metadata}
+ for m, e in store.get_backlinks(str(target))
+ ]
+ return _text_response({"path": str(target), "backlinks": out})
+
+ async def emit_edges(triples: list[dict]) -> ToolResponse:
+ """Finalize extraction. Emit the full set of triples for this file.
+
+ Call this exactly ONCE per file, with all triples you want to
+ commit. After this call, stop and produce a brief textual reply
+ summarizing what you emitted.
+
+ Args:
+ triples (list[dict]): List of {subject, predicate, object, confidence}.
+ `confidence` ∈ [0, 1]; values below the configured threshold
+ are dropped by the parser.
+ """
+ accepted = 0
+ rejected = 0
+ for t in triples:
+ try:
+ sink.append(_Triple.model_validate(t))
+ accepted += 1
+ except Exception:
+ rejected += 1
+ return _text_response({
+ "accepted": accepted,
+ "rejected": rejected,
+ "total_so_far": len(sink),
+ })
+
+ toolkit = Toolkit()
+ for fn in (
+ memory_get,
+ memory_list,
+ memory_resolve_wikilink,
+ memory_links,
+ memory_backlinks,
+ emit_edges,
+ ):
+ toolkit.register_tool_function(fn, namesake_strategy="override")
+ return toolkit, sink
+
+ def _sink_to_edges(self, sink: list[_Triple]) -> list[FileEdge]:
+ edges: list[FileEdge] = []
+ for tri in sink:
+ if tri.confidence < self.min_confidence:
+ continue
+ target = _strip_link_wrapper(tri.object)
+ if not target or target == _strip_link_wrapper(tri.subject):
+ continue
+ predicate = (
+ _normalize_predicate(tri.predicate)
+ if self.predicate_normalizer
+ else tri.predicate.strip()
+ )
+ edges.append(FileEdge(
+ target=target,
+ predicate=predicate or None,
+ source="llm",
+ confidence=tri.confidence,
+ ))
+ return edges
diff --git a/reme2/component/edge_extractor/regex_edge_extractor.py b/reme2/component/edge_extractor/regex_edge_extractor.py
new file mode 100644
index 00000000..c20811b2
--- /dev/null
+++ b/reme2/component/edge_extractor/regex_edge_extractor.py
@@ -0,0 +1,47 @@
+"""RegexEdgeExtractor — fast route per `structure.md` §"双态抽取".
+
+Pulls edges from inline syntax + frontmatter walk in O(text length).
+Synchronous and dependency-free; runs inside the watcher's hot path
+without blocking on any external service.
+
+Sources of edges:
+ - body inline syntax: `[[X]]`, `![[X#Section|Alias]]`,
+ `[predicate:: [[X]]]`, `(predicate:: [[X]])`
+ - frontmatter walk: `author: "[[John]]"` → predicate=author,
+ `related: ["[[X]]", "[[Y]]"]` → two edges
+
+Dedup key: (target, predicate, anchor, alias, embed). When the same
+edge appears in both body and frontmatter, the inline (regex) variant
+wins — it's the more direct authoring intent.
+"""
+
+from __future__ import annotations
+
+from .base_edge_extractor import BaseEdgeExtractor
+from ..component_registry import R
+from ...schema import FileEdge
+from ...utils.wikilink import parse_wikilinks, parse_wikilinks_from_metadata
+
+
+@R.register("regex")
+class RegexEdgeExtractor(BaseEdgeExtractor):
+ """Inline-syntax + frontmatter typed-edge extractor."""
+
+ async def extract(
+ self,
+ text: str,
+ metadata: dict | None = None,
+ path: str | None = None,
+ ) -> list[FileEdge]:
+ body_edges = parse_wikilinks(text or "")
+ meta_edges = parse_wikilinks_from_metadata(metadata or {})
+
+ seen: set[tuple] = set()
+ out: list[FileEdge] = []
+ for edge in (*body_edges, *meta_edges):
+ key = (edge.target, edge.predicate, edge.anchor, edge.alias, edge.embed)
+ if key in seen:
+ continue
+ seen.add(key)
+ out.append(edge)
+ return out
diff --git a/reme2/component/embedding/base_embedding_model.py b/reme2/component/embedding/base_embedding_model.py
index 21453bed..8ea7da96 100644
--- a/reme2/component/embedding/base_embedding_model.py
+++ b/reme2/component/embedding/base_embedding_model.py
@@ -26,17 +26,17 @@ class BaseEmbeddingModel(BaseComponent):
component_type = ComponentEnum.EMBEDDING_MODEL
def __init__(
- self,
- api_key: str | None = None,
- base_url: str | None = None,
- model_name: str = "",
- dimensions: int = 1024,
- pass_dimensions: bool = False,
- max_batch_size: int = 10,
- max_input_length: int = 8192,
- max_cache_size: int = 2000,
- enable_cache: bool = True,
- **kwargs,
+ self,
+ api_key: str | None = None,
+ base_url: str | None = None,
+ model_name: str = "",
+ dimensions: int = 1024,
+ pass_dimensions: bool = False,
+ max_batch_size: int = 10,
+ max_input_length: int = 8192,
+ max_cache_size: int = 2000,
+ enable_cache: bool = True,
+ **kwargs,
):
super().__init__(**kwargs)
self.api_key: str = api_key or os.environ.get("EMBEDDING_API_KEY", "")
@@ -81,7 +81,7 @@ class BaseEmbeddingModel(BaseComponent):
self.logger.warning(f"Embedding dim {actual_len} < expected {self.dimensions}, padding")
return embedding + [0.0] * (self.dimensions - actual_len)
self.logger.warning(f"Embedding dim {actual_len} > expected {self.dimensions}, truncating")
- return embedding[:self.dimensions]
+ return embedding[: self.dimensions]
def _get_cache_key(self, text: str) -> str:
"""Generate cache key from text + model_name + dimensions."""
@@ -109,7 +109,9 @@ class BaseEmbeddingModel(BaseComponent):
for key, emb in zip(data["keys"], data["embeddings"]):
emb_list = emb.tolist()
if len(emb_list) != self.dimensions:
- self.logger.warning(f"Cache dimension mismatch for {key}: expected {self.dimensions}, got {len(emb_list)}")
+ self.logger.warning(
+ f"Cache dimension mismatch for {key}: expected {self.dimensions}, got {len(emb_list)}",
+ )
continue
if len(self._embedding_cache) >= self.max_cache_size:
self.logger.info(f"Cache limit reached ({self.max_cache_size}), loaded {loaded_count}")
@@ -188,7 +190,7 @@ class BaseEmbeddingModel(BaseComponent):
async def get_embeddings(self, input_text: list[str], **kwargs) -> list[list[float] | None]:
"""Get embeddings for multiple texts with cache and batching."""
# TODO change to bytes instead of str
- truncated_texts = [t[:self.max_input_length] for t in input_text]
+ truncated_texts = [t[: self.max_input_length] for t in input_text]
results: list[list[float] | None] = [None] * len(truncated_texts)
texts_to_compute: list[tuple[int, str]] = []
@@ -202,7 +204,7 @@ class BaseEmbeddingModel(BaseComponent):
if texts_to_compute:
uncached_texts = [text for _, text in texts_to_compute]
for i in range(0, len(uncached_texts), self.max_batch_size):
- batch = texts_to_compute[i:i + self.max_batch_size]
+ batch = texts_to_compute[i : i + self.max_batch_size]
batch_indices = [idx for idx, _ in batch]
batch_texts = [text for _, text in batch]
diff --git a/reme2/component/file_parser/__init__.py b/reme2/component/file_parser/__init__.py
index de6491c7..17239557 100644
--- a/reme2/component/file_parser/__init__.py
+++ b/reme2/component/file_parser/__init__.py
@@ -1,11 +1,15 @@
"""File parser implementations for different file formats."""
from .base_file_parser import BaseFileParser
-from .default_file_parser import DefaultFileParser
from .md_file_parser import MdFileParser
+from .text_file_parser import TextFileParser
+
+# Backwards-compat alias.
+DefaultFileParser = TextFileParser
__all__ = [
"BaseFileParser",
"DefaultFileParser",
"MdFileParser",
+ "TextFileParser",
]
diff --git a/reme2/component/file_parser/base_file_parser.py b/reme2/component/file_parser/base_file_parser.py
index 580ee445..a79706fe 100644
--- a/reme2/component/file_parser/base_file_parser.py
+++ b/reme2/component/file_parser/base_file_parser.py
@@ -1,18 +1,98 @@
-"""Abstract base class for file parsers."""
+"""Abstract base class for file parsers.
+
+Single-pass parser: `parse(path, existing_chunks)` returns a `ParsedFile`
+carrying metadata + chunks (with embeddings attached) + edges.
+
+The parser owns the embedding pipeline. To avoid re-embedding unchanged
+blocks, the watcher passes in the file's prior chunks; the parser
+hash-diffs and only calls the embedding API for blocks whose hash is
+new. Vanished hashes simply drop out (the file_store's upsert is a
+delete-and-insert).
+
+If `existing_chunks` is None / empty, every chunk is embedded fresh.
+"""
from abc import abstractmethod
from ..base_component import BaseComponent
+from ..embedding import BaseEmbeddingModel
from ...enumeration import ComponentEnum, FileSuffixEnum
-from ...schema import FileChunk, FileMetadata
+from ...schema import FileChunk, ParsedFile
class BaseFileParser(BaseComponent):
- """Parser that declares handled suffixes and produces FileChunks."""
+ """Single-pass parser producing an embedded `ParsedFile`."""
component_type = ComponentEnum.FILE_PARSER
suffixes: list[FileSuffixEnum] = []
+ def __init__(self, embedding_model: str = "", **kwargs):
+ super().__init__(**kwargs)
+ self._embedding_model_name: str = embedding_model
+ self.embedding_model: BaseEmbeddingModel | None = None
+
+ async def _start(self) -> None:
+ if not self._embedding_model_name:
+ return
+ assert self.app_context is not None, "app_context must be provided"
+ models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
+ if self._embedding_model_name not in models:
+ raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
+ model = models[self._embedding_model_name]
+ if not isinstance(model, BaseEmbeddingModel):
+ raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
+ self.embedding_model = model
+
+ async def _close(self) -> None:
+ self.embedding_model = None
+
+ async def _embed_chunks(self, chunks: list[FileChunk]) -> list[FileChunk]:
+ """Attach embeddings to chunks if an embedding model is configured.
+
+ Failures (missing embeddings) leave `chunk.embedding=None`; the
+ file_store can still keyword-search them.
+ """
+ if not chunks or self.embedding_model is None:
+ return chunks
+ try:
+ await self.embedding_model.get_node_embeddings(chunks)
+ except Exception as e:
+ self.logger.warning(f"embedding chunks failed: {e}")
+ return chunks
+
+ @staticmethod
+ def _hash_diff_attach(
+ chunks: list[FileChunk],
+ existing_chunks: list[FileChunk] | None,
+ ) -> list[FileChunk]:
+ """Attach cached embeddings to chunks whose hash already exists.
+
+ Returns the dirty subset (chunks still needing embeddings). Mutates
+ the input chunks in place: matched chunks get their `embedding`
+ field set from the cache.
+ """
+ if not existing_chunks:
+ return list(chunks)
+ cached = {c.hash: c.embedding for c in existing_chunks if c.embedding}
+ if not cached:
+ return list(chunks)
+ dirty: list[FileChunk] = []
+ for c in chunks:
+ cached_emb = cached.get(c.hash)
+ if cached_emb is not None:
+ c.embedding = cached_emb
+ else:
+ dirty.append(c)
+ return dirty
+
@abstractmethod
- async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
- """Parse a file into metadata and chunks."""
+ async def parse(
+ self,
+ path: str,
+ existing_chunks: list[FileChunk] | None = None,
+ ) -> ParsedFile:
+ """Parse + embed the file, returning a fully populated `ParsedFile`.
+
+ `existing_chunks`, when provided, lets the parser reuse cached
+ embeddings for blocks whose hash hasn't changed.
+ """
diff --git a/reme2/component/file_parser/md_file_parser.py b/reme2/component/file_parser/md_file_parser.py
index fdb60373..7300e7b1 100644
--- a/reme2/component/file_parser/md_file_parser.py
+++ b/reme2/component/file_parser/md_file_parser.py
@@ -1,54 +1,226 @@
-"""Markdown file parser."""
+"""Markdown file parser — frontmatter + wikilink graph + AST-aware chunking.
-import asyncio
+The chunker splits markdown into semantic blocks using:
+ - ATX headings (#, ##, ..., ######) as section anchors
+ - Blank lines (paragraph boundaries) as soft splits within a section
+ - Code fences (``` or ~~~) preserved as a single block
+
+Each block carries a `heading_path` breadcrumb prepended to its text — gives
+the embedding model section context AND lets retrieval results show callers
+where the hit lives. The hash is computed over the final text (with
+breadcrumb), so renaming a heading correctly invalidates child block
+embeddings.
+
+Hash-diff cache compatibility: blocks with identical (heading_path + body)
+across edits produce the same hash, so the watcher can reuse old embeddings
+and only call the embedding API for dirty blocks.
+
+Edge extraction is delegated to a `BaseEdgeExtractor` resolved from the
+app context (`edge_extractor` constructor arg, defaults to "default"). If
+the app context isn't bound or the named extractor isn't registered, the
+parser falls back to a local `RegexEdgeExtractor` so unit tests / simple
+scripts can still parse without YAML wiring.
+"""
+
+import re
from pathlib import Path
import frontmatter
from .base_file_parser import BaseFileParser
from ..component_registry import R
-from ...enumeration import FileSuffixEnum
-from ...schema import FileChunk, FileMetadata
-from ...utils import chunk_markdown
+from ..edge_extractor import BaseEdgeExtractor, RegexEdgeExtractor
+from ...enumeration import ComponentEnum, FileSuffixEnum
+from ...schema import FileChunk, ParsedFile
+from ...utils import hash_text
@R.register("md")
class MdFileParser(BaseFileParser):
- """Parser for Markdown files with YAML frontmatter support."""
+ """Parser for Markdown files with YAML frontmatter and wikilink support."""
suffixes = [FileSuffixEnum.MD, FileSuffixEnum.MARKDOWN]
- def __init__(self, encoding: str = "utf-8", chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs):
+ _HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$")
+ _FENCE_RE = re.compile(r"^(```|~~~)")
+
+ def __init__(
+ self,
+ encoding: str = "utf-8",
+ edge_extractor: str = "default",
+ **kwargs,
+ ):
super().__init__(**kwargs)
self.encoding = encoding
- self.chunk_tokens = chunk_tokens
- self.chunk_overlap = chunk_overlap
+ self._edge_extractor_name: str = edge_extractor
+ self.edge_extractor: BaseEdgeExtractor | None = None
- async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
+ async def _start(self) -> None:
+ await super()._start()
+ # Try to resolve the configured extractor; fall back to a local
+ # regex extractor when no app_context / no matching component.
+ resolved: BaseEdgeExtractor | None = None
+ if self.app_context is not None and self._edge_extractor_name:
+ extractors = self.app_context.components.get(ComponentEnum.EDGE_EXTRACTOR, {})
+ candidate = extractors.get(self._edge_extractor_name)
+ if isinstance(candidate, BaseEdgeExtractor):
+ resolved = candidate
+ if resolved is None:
+ resolved = RegexEdgeExtractor()
+ await resolved.start()
+ self.edge_extractor = resolved
+
+ async def _close(self) -> None:
+ await super()._close()
+ self.edge_extractor = None
+
+ async def parse(
+ self,
+ path: str,
+ existing_chunks: list[FileChunk] | None = None,
+ ) -> ParsedFile:
file_path = Path(path)
+ raw = file_path.read_text(encoding=self.encoding)
+ post = frontmatter.loads(raw)
+ stat = file_path.stat()
+ metadata = dict(post.metadata)
+ content = post.content
+ absolute_path = str(file_path.absolute())
- def _read_and_parse():
- raw = file_path.read_text(encoding=self.encoding)
- post = frontmatter.loads(raw)
- stat = file_path.stat()
- return stat, dict(post.metadata), post.content
+ extractor = self.edge_extractor or RegexEdgeExtractor()
+ edges = await extractor.extract(content, metadata, path=absolute_path)
- stat, metadata, content = await asyncio.to_thread(_read_and_parse)
+ chunks = self.chunk_markdown(content, absolute_path)
+ dirty = self._hash_diff_attach(chunks, existing_chunks)
+ if dirty:
+ await self._embed_chunks(dirty)
- file_meta = FileMetadata(
- modified_time=stat.st_mtime,
- path=str(file_path.absolute()),
+ return ParsedFile(
+ file=file_path.stem,
+ path=absolute_path,
+ st_mtime=stat.st_mtime,
metadata=metadata,
+ edges=edges,
+ chunks=chunks,
)
- chunks = (
- chunk_markdown(
- content,
- file_meta.path,
- self.chunk_tokens,
- self.chunk_overlap,
- )
- or []
- )
+ # -- Chunker ----------------------------------------------------------
- return file_meta, chunks
+ @staticmethod
+ def _breadcrumb(heading_path: list[str]) -> str:
+ return " > ".join(heading_path) if heading_path else ""
+
+ @classmethod
+ def _make_block_text(cls, heading_path: list[str], body: str) -> str:
+ """Compose final block text: breadcrumb line (if any) + blank + body."""
+ body = body.rstrip("\n")
+ crumb = cls._breadcrumb(heading_path)
+ if crumb:
+ return f"{crumb}\n\n{body}" if body else crumb
+ return body
+
+ @classmethod
+ def chunk_markdown(cls, text: str, path: str) -> list[FileChunk]:
+ """Split markdown into AST-aware blocks (headings / paragraphs / fences)."""
+ if not text or not text.strip():
+ return []
+
+ lines = text.split("\n")
+ chunks: list[FileChunk] = []
+
+ heading_stack: list[tuple[int, str]] = [] # [(level, title)]
+ body_lines: list[str] = []
+ body_start = 1
+ in_fence = False
+ fence_marker = ""
+
+ def current_path() -> list[str]:
+ return [t for _, t in heading_stack]
+
+ def emit(block_text: str, start_line: int, end_line: int) -> None:
+ h = hash_text(block_text)
+ chunks.append(
+ FileChunk(
+ id=hash_text(f"{path}::{start_line}::{end_line}::{h}::{len(chunks)}"),
+ path=path,
+ start_line=start_line,
+ end_line=end_line,
+ text=block_text,
+ hash=h,
+ ),
+ )
+
+ def flush_body(end_line: int) -> None:
+ nonlocal body_lines, body_start
+ if not body_lines:
+ return
+ # Strip leading/trailing blank lines from the block (paragraph
+ # boundaries eat their own newline, but whitespace can sneak in
+ # via the fence path).
+ while body_lines and not body_lines[0].strip():
+ body_lines.pop(0)
+ body_start += 1
+ while body_lines and not body_lines[-1].strip():
+ body_lines.pop()
+ end_line -= 1
+ if not body_lines:
+ body_lines = []
+ return
+ raw_body = "\n".join(body_lines)
+ block_text = cls._make_block_text(current_path(), raw_body)
+ emit(block_text, body_start, end_line)
+ body_lines = []
+
+ for i, line in enumerate(lines, 1):
+ stripped = line.strip()
+
+ # Code fences: keep contents intact, no inner splits.
+ if not in_fence and cls._FENCE_RE.match(stripped):
+ if body_lines:
+ flush_body(end_line=i - 1)
+ in_fence = True
+ fence_marker = stripped[:3]
+ body_lines = [line]
+ body_start = i
+ continue
+ if in_fence:
+ body_lines.append(line)
+ if stripped.startswith(fence_marker):
+ in_fence = False
+ fence_marker = ""
+ flush_body(end_line=i)
+ continue
+
+ # ATX heading: closes prior block, opens a new section.
+ m = cls._HEADING_RE.match(line)
+ if m:
+ if body_lines:
+ flush_body(end_line=i - 1)
+ level = len(m.group(1))
+ title = m.group(2).strip()
+ heading_stack = [(lv, t) for lv, t in heading_stack if lv < level]
+ heading_stack.append((level, title))
+
+ # Heading line itself becomes a block (so the heading text is
+ # searchable as its own unit).
+ block_text = cls._make_block_text(current_path(), "")
+ emit(block_text, i, i)
+ body_start = i + 1
+ continue
+
+ # Blank line: paragraph boundary.
+ if not stripped:
+ if body_lines:
+ flush_body(end_line=i - 1)
+ body_start = i + 1
+ continue
+
+ # Regular content line.
+ if not body_lines:
+ body_start = i
+ body_lines.append(line)
+
+ if body_lines:
+ flush_body(end_line=len(lines))
+
+ return chunks
diff --git a/reme2/component/file_parser/text_file_parser.py b/reme2/component/file_parser/text_file_parser.py
index 490714e9..193cb38a 100644
--- a/reme2/component/file_parser/text_file_parser.py
+++ b/reme2/component/file_parser/text_file_parser.py
@@ -8,7 +8,7 @@ import aiofiles
from .base_file_parser import BaseFileParser
from ..component_registry import R
from ...enumeration import FileSuffixEnum
-from ...schema import FileChunk, FileMetadata
+from ...schema import FileChunk, ParsedFile
@R.register("text")
@@ -23,9 +23,14 @@ class TextFileParser(BaseFileParser):
self.chunk_size = max(32, chunk_tokens * 4)
self.overlap_size = max(0, chunk_overlap * 4)
- async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
+ async def parse(
+ self,
+ path: str,
+ existing_chunks: list[FileChunk] | None = None,
+ ) -> ParsedFile:
file_path = Path(path)
stat = file_path.stat()
+ absolute_path = str(file_path.absolute())
try:
async with aiofiles.open(file_path, encoding=self.encoding) as f:
@@ -34,17 +39,20 @@ class TextFileParser(BaseFileParser):
async with aiofiles.open(file_path, encoding=self.encoding, errors="ignore") as f:
content = await f.read()
except Exception:
- content = None
+ content = ""
- file_meta = FileMetadata(
+ chunks = self._chunk(content, absolute_path) if content else []
+ dirty = self._hash_diff_attach(chunks, existing_chunks)
+ if dirty:
+ await self._embed_chunks(dirty)
+
+ return ParsedFile(
file=file_path.stem,
- path=str(file_path.absolute()),
+ path=absolute_path,
st_mtime=stat.st_mtime,
+ chunks=chunks,
)
- chunks = self._chunk(content, file_meta.path) if content else []
- return file_meta, chunks
-
def _chunk(self, text: str, path: str) -> list[FileChunk]:
"""Split text into chunks with overlap."""
if not text.strip():
@@ -58,7 +66,7 @@ class TextFileParser(BaseFileParser):
for line_no, line in enumerate(lines, 1):
# Split long lines into segments
for start in range(0, max(1, len(line)), self.chunk_size):
- seg = line[start:start + self.chunk_size]
+ seg = line[start : start + self.chunk_size]
seg_chars = len(seg) + 1 # +1 for newline
# Flush when buffer would exceed limit
@@ -82,14 +90,16 @@ class TextFileParser(BaseFileParser):
h = hashlib.sha256(chunk_text.encode()).hexdigest()
chunk_id = hashlib.sha256(f"{path}:{start_line}:{end_line}:{h}:{len(chunks)}".encode()).hexdigest()
- chunks.append(FileChunk(
- id=chunk_id,
- path=path,
- start_line=start_line,
- end_line=end_line,
- text=chunk_text,
- hash=h,
- ))
+ chunks.append(
+ FileChunk(
+ id=chunk_id,
+ path=path,
+ start_line=start_line,
+ end_line=end_line,
+ text=chunk_text,
+ hash=h,
+ ),
+ )
def _carry_overlap(self, buf: list[tuple[str, int]]) -> tuple[list[tuple[str, int]], int]:
"""Keep overlapping lines from the end of buffer."""
diff --git a/reme2/component/file_store/__init__.py b/reme2/component/file_store/__init__.py
new file mode 100644
index 00000000..64d3c3b9
--- /dev/null
+++ b/reme2/component/file_store/__init__.py
@@ -0,0 +1,26 @@
+"""File store module.
+
+Unified storage for file metadata (frontmatter + mtime), the wikilink
+graph (edges per path), and chunks (text + embeddings) with vector /
+keyword / hybrid search.
+
+The store accepts a `ParsedFile` from the watcher's parsing pass via
+`upsert_parsed(...)`, dispatching meta + edges + chunks into the
+appropriate persistence pipelines.
+
+Two backends:
+ LocalFileStore — pure-Python with JSONL persistence (default,
+ zero deps, fine for small vaults).
+ SqliteFileStore — SQLite + FTS5 + sqlite-vec for keyword/vector
+ at scale; meta + edges live in relational tables.
+"""
+
+from .base_file_store import BaseFileStore
+from .local_file_store import LocalFileStore
+from .sqlite_file_store import SqliteFileStore
+
+__all__ = [
+ "BaseFileStore",
+ "LocalFileStore",
+ "SqliteFileStore",
+]
diff --git a/reme2/component/file_store/base_file_store.py b/reme2/component/file_store/base_file_store.py
new file mode 100644
index 00000000..c1493542
--- /dev/null
+++ b/reme2/component/file_store/base_file_store.py
@@ -0,0 +1,629 @@
+"""Abstract base class for file stores.
+
+A `FileStore` manages **everything** about a file in the vault:
+ - file metadata (frontmatter + mtime)
+ - the wikilink graph: edges per file (links / backlinks / resolution)
+ - graph-walk retrieval primitives (BFS expansion, decay scoring)
+ - chunks (text + embeddings + position) with vector / keyword / hybrid search
+
+Edges are stored independently of metadata in `_edges: dict[path,
+list[FileEdge]]`. The watcher submits a `ParsedFile` via
+`upsert_parsed(parsed)`; the store fans this out to meta + edge + chunk
+persistence in a single atomic-from-the-caller's-POV operation.
+
+Reads (`get_file_meta`, `get_links`, `resolve_wikilink`, `subgraph_score`,
+`filter`, ...) are **synchronous** — they hit an in-memory index that is
+rebuilt from the persistent backend on `_start`. Writes are **async** —
+they touch persistence first, then update the in-memory index, so on
+crash the on-disk view is the source of truth.
+
+Backends supply persistence for chunks, file metadata, AND edges via
+abstract methods. The default meta + edge persistence is sidecar JSONL;
+SQLite overrides with relational tables.
+"""
+
+import re
+from abc import abstractmethod
+from collections import defaultdict, deque
+from pathlib import Path
+from typing import Iterable
+
+from ..base_component import BaseComponent
+from ..embedding import BaseEmbeddingModel
+from ...enumeration import ComponentEnum
+from ...schema import ChunkFilter, FileChunk, FileEdge, FileMetadata, ParsedFile
+from ...utils.wikilink import extract_wikilinks
+
+
+def _target_stem(raw: str) -> str:
+ """Stem of a raw wikilink target — `"topics/X/X"` → `"X"`."""
+ target = raw.strip()
+ if target.endswith(".md"):
+ target = target[:-3]
+ return target.rsplit("/", 1)[-1]
+
+
+class BaseFileStore(BaseComponent):
+ """Unified file store: metadata + edges + chunks + graph + search."""
+
+ component_type = ComponentEnum.FILE_STORE
+
+ def __init__(
+ self,
+ store_name: str,
+ db_path: str | Path,
+ embedding_model: str = "default",
+ fts_enabled: bool = True,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self._embedding_model_name: str = embedding_model
+ self.embedding_model: BaseEmbeddingModel | None = None
+ self.store_name: str = store_name
+ self.db_path: Path = Path(db_path)
+ self.db_path.mkdir(parents=True, exist_ok=True)
+ self.vector_enabled: bool = bool(embedding_model)
+ self.fts_enabled: bool = fts_enabled
+
+ if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
+ raise ValueError(
+ f"Invalid store name '{store_name}'. Only alphanumeric characters and underscores are allowed.",
+ )
+ if not self.vector_enabled and not self.fts_enabled:
+ raise ValueError("At least one of embedding_model or fts_enabled must be set.")
+
+ # In-memory indices (rebuilt on _start from persisted state).
+ self._nodes: dict[str, FileMetadata] = {}
+ self._edges: dict[str, list[FileEdge]] = {}
+ self._stems: dict[str, set[str]] = defaultdict(set)
+ self._backlinks: dict[str, set[str]] = defaultdict(set)
+ self.vault_root: Path | None = None
+
+ # -- Lifecycle ----------------------------------------------------------
+
+ async def _start(self):
+ if self._embedding_model_name:
+ assert self.app_context is not None, "app_context must be provided"
+ models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
+ if self._embedding_model_name not in models:
+ raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
+ model = models[self._embedding_model_name]
+ if not isinstance(model, BaseEmbeddingModel):
+ raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
+ self.embedding_model = model
+ # Concrete backend's _start runs first (opens conn, creates tables);
+ # this base _start runs LAST so persistence is ready by the time we
+ # iterate it. Subclasses should `await super()._start()` at the END
+ # of their own _start.
+ await self._reload_file_metas()
+
+ async def _close(self):
+ self.embedding_model = None
+ self._nodes.clear()
+ self._edges.clear()
+ self._stems.clear()
+ self._backlinks.clear()
+
+ async def _reload_file_metas(self) -> None:
+ """Rebuild in-memory indices from persisted meta + edges. Called on _start."""
+ self._nodes.clear()
+ self._edges.clear()
+ self._stems.clear()
+ self._backlinks.clear()
+ count = 0
+ for meta in self._iter_persisted_metas():
+ self._nodes[meta.path] = meta
+ self._stems[Path(meta.path).stem].add(meta.path)
+ count += 1
+ edge_count = 0
+ for path, edges in self._iter_persisted_edges():
+ if path not in self._nodes:
+ continue # orphan — meta was already pruned
+ self._edges[path] = edges
+ for edge in edges:
+ self._backlinks[_target_stem(edge.target)].add(path)
+ edge_count += 1
+ self.logger.info(
+ f"[{self.store_name}] Loaded {count} file metas, {edge_count} edges into in-memory index",
+ )
+
+ # -- Embedding helpers --------------------------------------------------
+
+ @property
+ def embedding_dim(self) -> int:
+ return self.embedding_model.dimensions if self.embedding_model else 1024
+
+ def _disable_vector_search(self, reason: str = "embedding API error") -> None:
+ if self.vector_enabled:
+ self.logger.warning(f"[{self.store_name}] Disabling vector search: {reason}")
+ self.vector_enabled = False
+
+ async def _get_embeddings_safe(self, texts: list[str], **kwargs) -> list[list[float]] | None:
+ if not self.vector_enabled:
+ return None
+ try:
+ assert self.embedding_model is not None
+ return await self.embedding_model.get_embeddings(texts, **kwargs)
+ except Exception as e:
+ self._disable_vector_search(str(e))
+ return None
+
+ async def get_embedding(self, query: str, **kwargs) -> list[float] | None:
+ result = await self._get_embeddings_safe([query], **kwargs)
+ return result[0] if result else None
+
+ async def get_embeddings(self, queries: list[str], **kwargs) -> list[list[float]] | None:
+ return await self._get_embeddings_safe(queries, **kwargs)
+
+ # -- Vault root ---------------------------------------------------------
+
+ def set_vault_root(self, vault_root: str | Path | None) -> None:
+ """Bind the runtime vault root for explicit-path wikilink resolution."""
+ self.vault_root = Path(vault_root).resolve() if vault_root is not None else None
+
+ # -- File-meta CRUD (async writes; sync reads on in-memory index) -------
+
+ async def upsert_file_meta(self, meta: FileMetadata) -> None:
+ """Persist + index a file meta. Replaces any prior entry for the path.
+
+ Edges are NOT touched here — call `upsert_edges` separately, or
+ use `upsert_parsed` for the combined fan-out from a `ParsedFile`.
+ """
+ await self._persist_upsert_meta(meta)
+ self._nodes[meta.path] = meta
+ self._stems[Path(meta.path).stem].add(meta.path)
+
+ async def update_file_meta(self, path: str, **fields) -> FileMetadata | None:
+ """Patch an existing meta (model_copy + upsert). No-op if path unknown."""
+ existing = self._nodes.get(path)
+ if existing is None:
+ return None
+ updated = existing.model_copy(update=fields)
+ await self.upsert_file_meta(updated)
+ return updated
+
+ async def delete_file_meta(self, path: str) -> FileMetadata | None:
+ """Persist deletion + drop from index. Also drops the file's edges.
+
+ Returns prior meta if any.
+ """
+ await self._persist_delete_meta(path)
+ await self._persist_delete_edges(path)
+ existing = self._nodes.pop(path, None)
+ if existing is not None:
+ stem = Path(existing.path).stem
+ self._stems.get(stem, set()).discard(existing.path)
+ if not self._stems.get(stem):
+ self._stems.pop(stem, None)
+ for edge in self._edges.pop(path, []):
+ tgt = _target_stem(edge.target)
+ self._backlinks.get(tgt, set()).discard(path)
+ if not self._backlinks.get(tgt):
+ self._backlinks.pop(tgt, None)
+ return existing
+
+ def get_file_meta(self, path: str) -> FileMetadata | None:
+ return self._nodes.get(path)
+
+ @property
+ def nodes(self) -> dict[str, FileMetadata]:
+ return self._nodes
+
+ def __len__(self) -> int:
+ return len(self._nodes)
+
+ def __contains__(self, path: str) -> bool:
+ return path in self._nodes
+
+ # -- Edge CRUD ----------------------------------------------------------
+
+ async def upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
+ """Persist + reindex the edge list for `path`. Replaces any prior set."""
+ await self._persist_upsert_edges(path, edges)
+ # Drop old backlink contributions from this source.
+ for edge in self._edges.get(path, []):
+ tgt = _target_stem(edge.target)
+ self._backlinks.get(tgt, set()).discard(path)
+ if not self._backlinks.get(tgt):
+ self._backlinks.pop(tgt, None)
+ if edges:
+ self._edges[path] = list(edges)
+ for edge in edges:
+ self._backlinks[_target_stem(edge.target)].add(path)
+ else:
+ self._edges.pop(path, None)
+
+ def get_edges(self, path: str) -> list[FileEdge]:
+ """All edges originating from `path` (raw — not resolved)."""
+ return list(self._edges.get(path, ()))
+
+ async def upsert_parsed(self, parsed: ParsedFile) -> None:
+ """Single fan-out entry point for the watcher's parse pass.
+
+ Persists meta, edges, then chunks — the watcher gets one call and
+ the store handles the three sub-payloads in order.
+ """
+ await self.upsert_file_meta(FileMetadata(
+ file=parsed.file,
+ path=parsed.path,
+ st_mtime=parsed.st_mtime,
+ metadata=parsed.metadata,
+ ))
+ await self.upsert_edges(parsed.path, parsed.edges)
+ await self.upsert_chunks(parsed.path, parsed.chunks)
+
+ # -- Link queries (sync, in-memory) -------------------------------------
+
+ def get_links(self, path: str) -> list[tuple[FileMetadata, FileEdge]]:
+ """Files this `path` links TO (resolved only).
+
+ Returns a list of `(target_meta, edge)` pairs — predicate / anchor /
+ source / confidence are preserved on the edge so callers can reason
+ about typed graph structure. The same target file can appear multiple
+ times if linked via several edges (e.g. plain `[[X]]` plus
+ `[author:: [[X]]]`); caller dedupes by `target_meta.path` if needed.
+ """
+ if path not in self._nodes:
+ return []
+ out: list[tuple[FileMetadata, FileEdge]] = []
+ for edge in self._edges.get(path, ()):
+ hit = self.resolve_wikilink(edge.target)
+ if hit is not None and hit in self._nodes:
+ out.append((self._nodes[hit], edge))
+ return out
+
+ def get_backlinks(self, path: str) -> list[tuple[FileMetadata, FileEdge]]:
+ """Files that link TO `path` (resolved only).
+
+ Returns a list of `(source_meta, edge)` pairs — `edge` is the
+ specific FileEdge on the source file that resolves to `path`. A
+ single source can appear multiple times if it points at `path` via
+ multiple edges.
+ """
+ if path not in self._nodes:
+ return []
+ stem = Path(path).stem
+ candidates = self._backlinks.get(stem, set())
+ out: list[tuple[FileMetadata, FileEdge]] = []
+ for src in candidates:
+ src_meta = self._nodes.get(src)
+ if src_meta is None:
+ continue
+ for edge in self._edges.get(src, ()):
+ if self.resolve_wikilink(edge.target) == path:
+ out.append((src_meta, edge))
+ return out
+
+ def get_paths_by_stem(self, stem: str) -> list[str]:
+ return sorted(self._stems.get(stem, set()))
+
+ # -- Wikilink resolution ------------------------------------------------
+
+ def resolve_wikilink(self, wikilink: str) -> str | None:
+ target = wikilink.strip()
+ if not target:
+ return None
+
+ if "/" in target or target.endswith(".md"):
+ if self.vault_root is None:
+ return None
+ candidate = target if target.endswith(".md") else f"{target}.md"
+ abs_candidate = str((self.vault_root / candidate).resolve())
+ if abs_candidate in self._nodes:
+ return abs_candidate
+ return None
+
+ candidates = self.wikilink_candidates(target)
+ if len(candidates) == 1:
+ return candidates[0]
+ if len(candidates) > 1:
+ self.logger.warning(
+ f"Wikilink [[{target}]] is ambiguous, candidates: {candidates}",
+ )
+ return None
+
+ def resolve_wikilinks(self, wikilinks: list[str]) -> tuple[list[str], list[str]]:
+ """Return (resolved_paths, dangling_targets)."""
+ resolved: list[str] = []
+ dangling: list[str] = []
+ for link in wikilinks:
+ hit = self.resolve_wikilink(link)
+ if hit is None:
+ dangling.append(link)
+ else:
+ resolved.append(hit)
+ return resolved, dangling
+
+ def wikilink_candidates(self, target: str) -> list[str]:
+ """Paths `[[target]]` would resolve to (folder-note hit wins)."""
+ folder_hits = self._folder_notes(target)
+ if folder_hits:
+ return folder_hits
+ return self.get_paths_by_stem(target)
+
+ def collisions_after_create(self, proposed_path: str | Path) -> list[str]:
+ """Existing paths that would conflict with adding `proposed_path`."""
+ p = Path(proposed_path)
+ stem = p.stem
+ proposed_abs = str(p.resolve())
+ is_folder_note = p.parent.name == stem
+
+ existing_folder_notes = [path for path in self._folder_notes(stem) if path != proposed_abs]
+ existing_stems = [path for path in self.get_paths_by_stem(stem) if path != proposed_abs]
+
+ if is_folder_note:
+ return existing_folder_notes
+ return existing_folder_notes + [sp for sp in existing_stems if sp not in existing_folder_notes]
+
+ def all_ambiguous_wikilinks(self) -> dict[str, list[str]]:
+ """Stems whose `[[stem]]` form has >1 candidate."""
+ out: dict[str, list[str]] = {}
+ for stem in self._stems:
+ cands = self.wikilink_candidates(stem)
+ if len(cands) > 1:
+ out[stem] = cands
+ return out
+
+ def _folder_notes(self, stem: str) -> list[str]:
+ return sorted(p for p in self._stems.get(stem, ()) if Path(p).parent.name == stem)
+
+ # -- Graph-walk retrieval primitives ------------------------------------
+
+ def expand_neighbors(
+ self,
+ seed_paths: Iterable[str],
+ depth: int = 1,
+ direction: str = "both",
+ per_node_cap: int | None = 50,
+ ) -> dict[str, int]:
+ """BFS expansion over wikilink edges. Returns `{path: hop_distance}`."""
+ if direction not in ("out", "in", "both"):
+ raise ValueError(f"direction must be one of out/in/both, got {direction!r}")
+ if depth < 0:
+ raise ValueError(f"depth must be >= 0, got {depth}")
+
+ seen: dict[str, int] = {}
+ frontier: deque[tuple[str, int]] = deque()
+ for path in seed_paths:
+ if path in self._nodes and path not in seen:
+ seen[path] = 0
+ frontier.append((path, 0))
+
+ while frontier:
+ path, dist = frontier.popleft()
+ if dist >= depth:
+ continue
+
+ neighbors: list[str] = []
+ if direction in ("out", "both"):
+ neighbors.extend(m.path for m, _ in self.get_links(path))
+ if direction in ("in", "both"):
+ neighbors.extend(m.path for m, _ in self.get_backlinks(path))
+
+ if per_node_cap is not None and len(neighbors) > per_node_cap:
+ neighbors = neighbors[:per_node_cap]
+
+ for nb in neighbors:
+ if nb in seen:
+ continue
+ seen[nb] = dist + 1
+ frontier.append((nb, dist + 1))
+
+ return seen
+
+ def subgraph_score(
+ self,
+ seed_paths: Iterable[str],
+ decay: float = 0.5,
+ depth: int = 1,
+ direction: str = "both",
+ per_node_cap: int | None = 50,
+ ) -> dict[str, float]:
+ """Decayed score per path: seed=1.0, 1-hop=decay, 2-hop=decay²..."""
+ if not (0.0 <= decay <= 1.0):
+ raise ValueError(f"decay must be in [0, 1], got {decay}")
+ hops = self.expand_neighbors(seed_paths, depth, direction, per_node_cap)
+ return {path: decay ** hop for path, hop in hops.items()}
+
+ def extract_anchor_paths(self, text: str) -> list[str]:
+ """Pull `[[X]]` anchors from `text` and resolve each (deduped)."""
+ seen: set[str] = set()
+ out: list[str] = []
+ for raw in extract_wikilinks(text):
+ hit = self.resolve_wikilink(raw)
+ if hit is not None and hit not in seen:
+ seen.add(hit)
+ out.append(hit)
+ return out
+
+ # -- Filter compilation -------------------------------------------------
+
+ def filter(
+ self,
+ paths: list[str] | None = None,
+ tags: list[str] | None = None,
+ exclude_paths: list[str] | None = None,
+ ) -> ChunkFilter:
+ """Compile user-facing intent into a path-set filter for chunk search."""
+ cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
+ if cf.is_empty():
+ return cf
+ cf.resolved_paths = {p for p, m in self._nodes.items() if cf.match_metadata(p, m.metadata)}
+ return cf
+
+ # -- Keyword scoring utility --------------------------------------------
+
+ @staticmethod
+ def _score_keyword_match(query: str, text: str) -> float:
+ words = query.split()
+ if not words:
+ return 0.0
+ query_lower = query.lower()
+ words_lower = [w.lower() for w in words]
+ text_lower = text.lower()
+ n_words = len(words)
+
+ match_count = sum(1 for w in words_lower if w in text_lower)
+ if match_count == 0:
+ return 0.0
+
+ base_score = match_count / n_words
+ phrase_bonus = 0.2 if n_words > 1 and query_lower in text_lower else 0.0
+ return min(1.0, base_score + phrase_bonus)
+
+ # -- Filter utility (chunk side) ----------------------------------------
+
+ @staticmethod
+ def _apply_filter(chunks: list[FileChunk], chunk_filter: ChunkFilter | None) -> list[FileChunk]:
+ if chunk_filter is None or chunk_filter.resolved_paths is None:
+ return chunks
+ return [c for c in chunks if chunk_filter.match_path(c.path)]
+
+ # -- Default file-meta persistence (sidecar JSONL) ----------------------
+
+ @property
+ def _file_metas_path(self) -> Path:
+ return self.db_path / f"{self.store_name}_files.jsonl"
+
+ async def _persist_upsert_meta(self, meta: FileMetadata) -> None:
+ """Default: full rewrite of the sidecar jsonl. Backends with proper
+ relational storage (sqlite) override this for per-row UPSERT."""
+ # Snapshot current in-memory state with the new entry merged in.
+ # Note: in-memory not yet updated by caller — we add `meta` ourselves.
+ snapshot = {**self._nodes, meta.path: meta}
+ self._write_metas_jsonl(snapshot.values())
+
+ async def _persist_delete_meta(self, path: str) -> None:
+ """Default: full rewrite of the sidecar jsonl, omitting `path`."""
+ snapshot = {p: m for p, m in self._nodes.items() if p != path}
+ self._write_metas_jsonl(snapshot.values())
+
+ def _iter_persisted_metas(self) -> Iterable[FileMetadata]:
+ """Default: read all rows from the sidecar jsonl."""
+ path = self._file_metas_path
+ if not path.exists():
+ return []
+ out: list[FileMetadata] = []
+ for line in path.read_text(encoding="utf-8").splitlines():
+ if not line.strip():
+ continue
+ try:
+ out.append(FileMetadata.model_validate_json(line))
+ except Exception as e:
+ self.logger.warning(f"Bad row in {path}: {e}")
+ return out
+
+ def _write_metas_jsonl(self, metas: Iterable[FileMetadata]) -> None:
+ path = self._file_metas_path
+ lines = [m.model_dump_json() for m in metas]
+ content = "\n".join(lines)
+ tmp = path.with_suffix(".tmp")
+ try:
+ tmp.write_text(content, encoding="utf-8")
+ tmp.replace(path)
+ except Exception as e:
+ self.logger.error(f"Failed to write {path}: {e}")
+ raise
+ finally:
+ if tmp.exists():
+ tmp.unlink()
+
+ # -- Default edge persistence (sidecar JSONL) ---------------------------
+
+ @property
+ def _edges_path(self) -> Path:
+ return self.db_path / f"{self.store_name}_edges.jsonl"
+
+ async def _persist_upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
+ """Default: full rewrite of the edge sidecar with `path`'s row replaced."""
+ snapshot = dict(self._edges)
+ if edges:
+ snapshot[path] = list(edges)
+ else:
+ snapshot.pop(path, None)
+ self._write_edges_jsonl(snapshot)
+
+ async def _persist_delete_edges(self, path: str) -> None:
+ """Default: full rewrite of the edge sidecar omitting `path`."""
+ snapshot = {p: e for p, e in self._edges.items() if p != path}
+ self._write_edges_jsonl(snapshot)
+
+ def _iter_persisted_edges(self) -> Iterable[tuple[str, list[FileEdge]]]:
+ """Default: yield (path, edges) rows from the sidecar jsonl."""
+ path = self._edges_path
+ if not path.exists():
+ return []
+ out: list[tuple[str, list[FileEdge]]] = []
+ import json as _json
+ for line in path.read_text(encoding="utf-8").splitlines():
+ if not line.strip():
+ continue
+ try:
+ row = _json.loads(line)
+ edges = [FileEdge.model_validate(e) for e in row.get("edges", [])]
+ out.append((row["path"], edges))
+ except Exception as e:
+ self.logger.warning(f"Bad edge row in {path}: {e}")
+ return out
+
+ def _write_edges_jsonl(self, edges_by_path: dict[str, list[FileEdge]]) -> None:
+ import json as _json
+ path = self._edges_path
+ lines = [
+ _json.dumps(
+ {"path": p, "edges": [e.model_dump(exclude_none=True) for e in edges]},
+ ensure_ascii=False,
+ )
+ for p, edges in edges_by_path.items()
+ ]
+ content = "\n".join(lines)
+ tmp = path.with_suffix(".tmp")
+ try:
+ tmp.write_text(content, encoding="utf-8")
+ tmp.replace(path)
+ except Exception as e:
+ self.logger.error(f"Failed to write {path}: {e}")
+ raise
+ finally:
+ if tmp.exists():
+ tmp.unlink()
+
+ # -- Abstract chunk APIs ------------------------------------------------
+
+ @abstractmethod
+ async def clear_all(self):
+ """Clear all indexed data (chunks + file metas)."""
+
+ @abstractmethod
+ async def upsert_chunks(self, path: str, chunks: list[FileChunk]):
+ """Insert or update all chunks for a file path. Embeddings pre-attached."""
+
+ @abstractmethod
+ async def delete_chunks(self, path: str):
+ """Delete all chunks for a file path."""
+
+ @abstractmethod
+ async def get_chunks(self, path: str) -> list[FileChunk]:
+ """All chunks for a file path."""
+
+ @abstractmethod
+ async def get_chunks_by_paths(self, paths: Iterable[str]) -> list[FileChunk]:
+ """Batch fetch chunks across many paths (used by graph-walk retrieval)."""
+
+ @abstractmethod
+ async def vector_search(
+ self,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
+ ) -> list[FileChunk]:
+ """Vector similarity search."""
+
+ @abstractmethod
+ async def keyword_search(
+ self,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
+ ) -> list[FileChunk]:
+ """Full-text/keyword search."""
diff --git a/reme2/component/chunk_store/local_chunk_store.py b/reme2/component/file_store/local_file_store.py
similarity index 64%
rename from reme2/component/chunk_store/local_chunk_store.py
rename to reme2/component/file_store/local_file_store.py
index 062758e8..8a95b442 100644
--- a/reme2/component/chunk_store/local_chunk_store.py
+++ b/reme2/component/file_store/local_file_store.py
@@ -1,19 +1,32 @@
-"""Pure-Python chunk storage with JSONL persistence."""
+"""Pure-Python in-memory store with on-close JSONL persistence.
+
+Design (per project clarification):
+ - During runtime, ALL state lives in memory — meta + edges + chunks.
+ Per-write disk I/O is suppressed; `_persist_upsert_meta` / `_persist_upsert_edges`
+ are no-ops so each upsert_* costs only a dict mutation.
+ - On `_start`, load the JSONL sidecars into memory.
+ - On `_close`, flush the full in-memory snapshot back to JSONL.
+
+Trade-off: lower write latency, but a hard crash drops anything since
+the last clean shutdown. For larger / write-critical workloads, use
+`SqliteFileStore` instead (per-write transaction).
+"""
import json
from pathlib import Path
+from typing import Iterable
import numpy as np
-from .base_chunk_store import BaseChunkStore
+from .base_file_store import BaseFileStore
from ..component_registry import R
-from ...schema import ChunkFilter, FileChunk
+from ...schema import ChunkFilter, FileChunk, FileEdge, FileMetadata
from ...utils import batch_cosine_similarity
@R.register("local")
-class LocalChunkStore(BaseChunkStore):
- """In-memory chunk storage with JSONL disk persistence."""
+class LocalFileStore(BaseFileStore):
+ """In-memory chunk + meta + edge store with deferred JSONL persistence."""
def __init__(self, encoding: str = "utf-8", **kwargs):
super().__init__(**kwargs)
@@ -24,7 +37,6 @@ class LocalChunkStore(BaseChunkStore):
# -- Persistence helpers ------------------------------------------------
async def _load_chunks(self) -> None:
- """Load chunks from JSONL file into memory."""
if not self._chunks_file.exists():
return
try:
@@ -39,7 +51,6 @@ class LocalChunkStore(BaseChunkStore):
self.logger.warning(f"Failed to load chunks: {e}")
async def _save_chunks(self) -> None:
- """Persist chunks to JSONL file with atomic write."""
lines = [json.dumps(c.model_dump(mode="json"), ensure_ascii=False) for c in self._chunks.values()]
content = "\n".join(lines)
temp_path = self._chunks_file.with_suffix(".tmp")
@@ -56,30 +67,52 @@ class LocalChunkStore(BaseChunkStore):
# -- Lifecycle ----------------------------------------------------------
async def _start(self) -> None:
- """Load persisted data into memory."""
await self._load_chunks()
- self.logger.info(f"LocalChunkStore '{self.store_name}' ready: {len(self._chunks)} chunks")
+ # Base loads metas + edges from sidecar JSONL via the default
+ # _iter_persisted_metas / _iter_persisted_edges implementations.
await super()._start()
+ self.logger.info(
+ f"LocalFileStore '{self.store_name}' ready: "
+ f"{len(self._nodes)} files, {sum(len(v) for v in self._edges.values())} edges, "
+ f"{len(self._chunks)} chunks",
+ )
async def _close(self) -> None:
- """Flush state to disk and clear memory."""
- await self._save_chunks()
+ # Flush the full in-memory snapshot before tearing down state.
+ try:
+ await self._save_chunks()
+ self._write_metas_jsonl(self._nodes.values())
+ self._write_edges_jsonl(self._edges)
+ except Exception as e:
+ self.logger.error(f"Failed to flush LocalFileStore '{self.store_name}': {e}")
self._chunks.clear()
await super()._close()
+ # -- Persistence overrides: no-op (we flush on close) -------------------
+
+ async def _persist_upsert_meta(self, meta: FileMetadata) -> None:
+ pass
+
+ async def _persist_delete_meta(self, path: str) -> None:
+ pass
+
+ async def _persist_upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
+ pass
+
+ async def _persist_delete_edges(self, path: str) -> None:
+ pass
+
# -- Write operations ---------------------------------------------------
async def upsert_chunks(self, path: str, chunks: list[FileChunk]) -> None:
- """Insert or update a file and its chunks."""
+ """Insert or update a file's chunks. Chunks arrive embedded."""
await self.delete_chunks(path)
if not chunks:
return
- chunks = await self.get_chunk_embeddings(chunks)
for chunk in chunks:
self._chunks[chunk.id] = chunk
async def delete_chunks(self, path: str) -> None:
- """Delete a file and all its chunks."""
to_delete = [cid for cid, chunk in self._chunks.items() if chunk.path == path]
for cid in to_delete:
del self._chunks[cid]
@@ -87,20 +120,26 @@ class LocalChunkStore(BaseChunkStore):
# -- Read operations ----------------------------------------------------
async def get_chunks(self, path: str) -> list[FileChunk]:
- """Get all chunks for a file, sorted by start_line."""
chunks = [chunk for chunk in self._chunks.values() if chunk.path == path]
chunks.sort(key=lambda c: c.start_line)
return chunks
+ async def get_chunks_by_paths(self, paths: Iterable[str]) -> list[FileChunk]:
+ wanted = set(paths)
+ if not wanted:
+ return []
+ chunks = [c for c in self._chunks.values() if c.path in wanted]
+ chunks.sort(key=lambda c: (c.path, c.start_line))
+ return chunks
+
# -- Search operations --------------------------------------------------
async def vector_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
+ self,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
- """Cosine-similarity vector search over in-memory embeddings."""
if not self.vector_enabled or not query:
return []
@@ -147,12 +186,11 @@ class LocalChunkStore(BaseChunkStore):
return results[:limit]
async def keyword_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
+ self,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
- """Keyword search via substring matching."""
if not self.fts_enabled or not query:
return []
@@ -182,7 +220,13 @@ class LocalChunkStore(BaseChunkStore):
return results[:limit]
async def clear_all(self) -> None:
- """Clear all indexed data from memory and disk."""
+ """Clear all in-memory state and on-disk JSONL sidecars."""
self._chunks.clear()
+ self._nodes.clear()
+ self._edges.clear()
+ self._stems.clear()
+ self._backlinks.clear()
await self._save_chunks()
- self.logger.info(f"Cleared all data from LocalChunkStore '{self.store_name}'")
+ self._write_metas_jsonl([])
+ self._write_edges_jsonl({})
+ self.logger.info(f"Cleared all data from LocalFileStore '{self.store_name}'")
diff --git a/reme2/component/chunk_store/sqlite_chunk_store.py b/reme2/component/file_store/sqlite_file_store.py
similarity index 58%
rename from reme2/component/chunk_store/sqlite_chunk_store.py
rename to reme2/component/file_store/sqlite_file_store.py
index bb01dd86..2fa851eb 100644
--- a/reme2/component/chunk_store/sqlite_chunk_store.py
+++ b/reme2/component/file_store/sqlite_file_store.py
@@ -1,22 +1,24 @@
-"""SQLite chunk storage backend."""
+"""SQLite file store backend (chunks + file metadata)."""
import json
import sqlite3
import struct
import time
+from typing import Iterable
-from .base_chunk_store import BaseChunkStore
+from .base_file_store import BaseFileStore
from ..component_registry import R
-from ...schema import ChunkFilter, FileChunk
+from ...schema import ChunkFilter, FileChunk, FileEdge, FileMetadata
@R.register("sqlite")
-class SqliteChunkStore(BaseChunkStore):
- """SQLite chunk storage with vector and full-text search.
+class SqliteFileStore(BaseFileStore):
+ """SQLite file store with vector + full-text search.
Uses sqlite-vec for vector similarity search and FTS5 with trigram
- tokenizer for keyword search. Falls back to LIKE-based substring
- search for short query terms.
+ tokenizer for keyword search. File metadata lives in a relational
+ table (`files_{store_name}`) with native UPSERT — overrides the
+ base class's default JSONL sidecar.
"""
def __init__(self, vec_ext_path: str = "", **kwargs):
@@ -38,13 +40,21 @@ class SqliteChunkStore(BaseChunkStore):
def fts_table(self) -> str:
return f"chunks_fts_{self.store_name}"
+ @property
+ def files_table(self) -> str:
+ return f"files_{self.store_name}"
+
+ @property
+ def edges_table(self) -> str:
+ return f"edges_{self.store_name}"
+
@staticmethod
def vector_to_blob(embedding: list[float]) -> bytes:
return struct.pack(f"{len(embedding)}f", *embedding)
# -- Lifecycle ----------------------------------------------------------
- async def _start(self, app_context=None) -> None:
+ async def _start(self) -> None:
self.conn = sqlite3.connect(self.db_path / "reme.db", check_same_thread=False)
if self.vector_enabled:
@@ -84,13 +94,14 @@ class SqliteChunkStore(BaseChunkStore):
self.conn.enable_load_extension(False)
await self._create_tables()
- self.logger.info(f"SqliteChunkStore '{self.store_name}' ready: db={self.db_path / 'reme.db'}")
- await super()._start(app_context)
+ self.logger.info(f"SqliteFileStore '{self.store_name}' ready: db={self.db_path / 'reme.db'}")
+ await super()._start()
async def _create_tables(self) -> None:
cursor = self.conn.cursor()
try:
- cursor.execute(f"""
+ cursor.execute(
+ f"""
CREATE TABLE IF NOT EXISTS {self.chunks_table} (
id TEXT PRIMARY KEY,
path TEXT,
@@ -101,23 +112,26 @@ class SqliteChunkStore(BaseChunkStore):
embedding TEXT,
updated_at INTEGER
)
- """)
+ """,
+ )
cursor.execute(
- f"CREATE INDEX IF NOT EXISTS idx_{self.chunks_table}_path "
- f"ON {self.chunks_table}(path)",
+ f"CREATE INDEX IF NOT EXISTS idx_{self.chunks_table}_path " f"ON {self.chunks_table}(path)",
)
if self.vector_enabled:
- cursor.execute(f"""
+ cursor.execute(
+ f"""
CREATE VIRTUAL TABLE IF NOT EXISTS {self.vector_table} USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[{self.embedding_dim}]
)
- """)
+ """,
+ )
self.logger.info(f"Created vector table (dims={self.embedding_dim})")
if self.fts_enabled:
- cursor.execute(f"""
+ cursor.execute(
+ f"""
CREATE VIRTUAL TABLE IF NOT EXISTS {self.fts_table} USING fts5(
text,
id UNINDEXED,
@@ -126,9 +140,30 @@ class SqliteChunkStore(BaseChunkStore):
end_line UNINDEXED,
tokenize='trigram'
)
- """)
+ """,
+ )
self.logger.info("Created FTS5 table with trigram tokenizer")
+ cursor.execute(
+ f"""
+ CREATE TABLE IF NOT EXISTS {self.files_table} (
+ path TEXT PRIMARY KEY,
+ file TEXT,
+ st_mtime REAL,
+ metadata TEXT
+ )
+ """,
+ )
+
+ cursor.execute(
+ f"""
+ CREATE TABLE IF NOT EXISTS {self.edges_table} (
+ path TEXT PRIMARY KEY,
+ edges TEXT
+ )
+ """,
+ )
+
self.conn.commit()
except Exception as e:
self.logger.error(f"Failed to create tables: {e}")
@@ -150,7 +185,8 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute("BEGIN")
old_ids = [
- row[0] for row in cursor.execute(
+ row[0]
+ for row in cursor.execute(
f"SELECT id FROM {self.chunks_table} WHERE path = ?",
(path,),
).fetchall()
@@ -165,7 +201,6 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute(f"DELETE FROM {self.fts_table} WHERE path = ?", (path,))
if chunks:
- chunks = await self.get_chunk_embeddings(chunks)
now = int(time.time() * 1000)
for chunk in chunks:
cursor.execute(
@@ -212,8 +247,10 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute("BEGIN")
chunk_ids = [
- row[0] for row in cursor.execute(
- f"SELECT id FROM {self.chunks_table} WHERE path = ?", (path,),
+ row[0]
+ for row in cursor.execute(
+ f"SELECT id FROM {self.chunks_table} WHERE path = ?",
+ (path,),
).fetchall()
]
@@ -254,15 +291,17 @@ class SqliteChunkStore(BaseChunkStore):
embedding = json.loads(emb_str)
except (json.JSONDecodeError, TypeError):
pass
- chunks.append(FileChunk(
- id=chunk_id,
- path=path_val,
- start_line=start,
- end_line=end,
- text=text,
- hash=hash_val,
- embedding=embedding,
- ))
+ chunks.append(
+ FileChunk(
+ id=chunk_id,
+ path=path_val,
+ start_line=start,
+ end_line=end,
+ text=text,
+ hash=hash_val,
+ embedding=embedding,
+ ),
+ )
return chunks
except Exception as e:
self.logger.error(f"Failed to get chunks for {path}: {e}")
@@ -270,13 +309,62 @@ class SqliteChunkStore(BaseChunkStore):
finally:
cursor.close()
+ async def get_chunks_by_paths(self, paths: Iterable[str]) -> list[FileChunk]:
+ """Batch fetch chunks across many paths.
+
+ Splits into ≤900-placeholder batches to stay under sqlite's
+ SQLITE_MAX_VARIABLE_NUMBER (default 999, leave headroom).
+ """
+ wanted = list({p for p in paths})
+ if not wanted:
+ return []
+
+ cursor = self.conn.cursor()
+ try:
+ chunks: list[FileChunk] = []
+ BATCH = 900
+ for offset in range(0, len(wanted), BATCH):
+ batch = wanted[offset : offset + BATCH]
+ placeholders = ",".join("?" * len(batch))
+ cursor.execute(
+ f"""SELECT id, path, start_line, end_line, text, hash, embedding
+ FROM {self.chunks_table} WHERE path IN ({placeholders})
+ ORDER BY path, start_line""",
+ batch,
+ )
+ for row in cursor.fetchall():
+ chunk_id, path_val, start_line, end, text, hash_val, emb_str = row
+ embedding = None
+ if emb_str:
+ try:
+ embedding = json.loads(emb_str)
+ except (json.JSONDecodeError, TypeError):
+ pass
+ chunks.append(
+ FileChunk(
+ id=chunk_id,
+ path=path_val,
+ start_line=start_line,
+ end_line=end,
+ text=text,
+ hash=hash_val,
+ embedding=embedding,
+ ),
+ )
+ return chunks
+ except Exception as e:
+ self.logger.error(f"Failed to batch get chunks ({len(wanted)} paths): {e}")
+ return []
+ finally:
+ cursor.close()
+
# -- Search helpers -----------------------------------------------------
@staticmethod
def _sanitize_fts_query(query: str) -> str:
if not query:
return ""
- special_chars = list('*?:^()[]{}\'"`|+-=<>!@#$%&\\/,;')
+ special_chars = list("*?:^()[]{}'\"`|+-=<>!@#$%&\\/,;")
cleaned = query
for ch in special_chars:
cleaned = cleaned.replace(ch, " ")
@@ -296,10 +384,10 @@ class SqliteChunkStore(BaseChunkStore):
# -- Search operations --------------------------------------------------
async def vector_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
+ self,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
if not self.vector_enabled or not query:
return []
@@ -325,15 +413,17 @@ class SqliteChunkStore(BaseChunkStore):
chunks = []
for cid, path, start, end, text, dist in cursor.fetchall():
score = max(0.0, 1.0 - dist / 2.0)
- chunks.append(FileChunk(
- id=cid,
- path=path,
- start_line=start,
- end_line=end,
- text=text,
- hash="",
- scores={"vector": score, "score": score},
- ))
+ chunks.append(
+ FileChunk(
+ id=cid,
+ path=path,
+ start_line=start,
+ end_line=end,
+ text=text,
+ hash="",
+ scores={"vector": score, "score": score},
+ ),
+ )
chunks = self._apply_filter(chunks, chunk_filter)
chunks.sort(key=lambda c: c.score, reverse=True)
@@ -345,10 +435,10 @@ class SqliteChunkStore(BaseChunkStore):
cursor.close()
async def keyword_search(
- self,
- query: str,
- limit: int,
- chunk_filter: ChunkFilter | None = None,
+ self,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
if not self.fts_enabled or not query:
return []
@@ -391,15 +481,17 @@ class SqliteChunkStore(BaseChunkStore):
chunks = []
for cid, path, start, end, text, rank in cursor.fetchall():
score = max(0.0, 1.0 / (1.0 + abs(rank)))
- chunks.append(FileChunk(
- id=cid,
- path=path,
- start_line=start,
- end_line=end,
- text=text,
- hash="",
- scores={"keyword": score, "score": score},
- ))
+ chunks.append(
+ FileChunk(
+ id=cid,
+ path=path,
+ start_line=start,
+ end_line=end,
+ text=text,
+ hash="",
+ scores={"keyword": score, "score": score},
+ ),
+ )
chunks.sort(key=lambda c: c.score, reverse=True)
return chunks
except Exception as e:
@@ -437,15 +529,17 @@ class SqliteChunkStore(BaseChunkStore):
if score == 0.0:
continue
- chunks.append(FileChunk(
- id=cid,
- path=path,
- start_line=start,
- end_line=end,
- text=text,
- hash="",
- scores={"keyword": score, "score": score},
- ))
+ chunks.append(
+ FileChunk(
+ id=cid,
+ path=path,
+ start_line=start,
+ end_line=end,
+ text=text,
+ hash="",
+ scores={"keyword": score, "score": score},
+ ),
+ )
chunks.sort(key=lambda c: c.score, reverse=True)
return chunks[:limit]
@@ -466,6 +560,8 @@ class SqliteChunkStore(BaseChunkStore):
cursor.execute(f"DELETE FROM {self.vector_table}")
if self.fts_enabled:
cursor.execute(f"DELETE FROM {self.fts_table}")
+ cursor.execute(f"DELETE FROM {self.files_table}")
+ cursor.execute(f"DELETE FROM {self.edges_table}")
cursor.execute("COMMIT")
except Exception as e:
cursor.execute("ROLLBACK")
@@ -473,4 +569,124 @@ class SqliteChunkStore(BaseChunkStore):
raise
finally:
cursor.close()
- self.logger.info(f"Cleared all data from SqliteChunkStore '{self.store_name}'")
+ self.logger.info(f"Cleared all data from SqliteFileStore '{self.store_name}'")
+
+ # -- File-meta persistence (overrides default JSONL sidecar) ------------
+
+ async def _persist_upsert_meta(self, meta: FileMetadata) -> None:
+ cursor = self.conn.cursor()
+ try:
+ cursor.execute(
+ f"""INSERT INTO {self.files_table}
+ (path, file, st_mtime, metadata)
+ VALUES (?, ?, ?, ?)
+ ON CONFLICT(path) DO UPDATE SET
+ file = excluded.file,
+ st_mtime = excluded.st_mtime,
+ metadata = excluded.metadata""",
+ (
+ meta.path,
+ meta.file,
+ meta.st_mtime,
+ json.dumps(meta.metadata, ensure_ascii=False, default=str),
+ ),
+ )
+ self.conn.commit()
+ except Exception as e:
+ self.conn.rollback()
+ self.logger.error(f"Failed to upsert file meta {meta.path}: {e}")
+ raise
+ finally:
+ cursor.close()
+
+ async def _persist_delete_meta(self, path: str) -> None:
+ cursor = self.conn.cursor()
+ try:
+ cursor.execute(f"DELETE FROM {self.files_table} WHERE path = ?", (path,))
+ self.conn.commit()
+ except Exception as e:
+ self.conn.rollback()
+ self.logger.error(f"Failed to delete file meta {path}: {e}")
+ raise
+ finally:
+ cursor.close()
+
+ def _iter_persisted_metas(self) -> Iterable[FileMetadata]:
+ if self.conn is None:
+ return []
+ cursor = self.conn.cursor()
+ try:
+ cursor.execute(
+ f"SELECT path, file, st_mtime, metadata "
+ f"FROM {self.files_table}",
+ )
+ out: list[FileMetadata] = []
+ for path, file, st_mtime, metadata_json in cursor.fetchall():
+ try:
+ out.append(
+ FileMetadata(
+ path=path,
+ file=file or "",
+ st_mtime=st_mtime or 0.0,
+ metadata=json.loads(metadata_json) if metadata_json else {},
+ ),
+ )
+ except Exception as e:
+ self.logger.warning(f"Bad file meta row for {path}: {e}")
+ return out
+ finally:
+ cursor.close()
+
+ # -- Edge persistence (overrides default JSONL sidecar) -----------------
+
+ async def _persist_upsert_edges(self, path: str, edges: list[FileEdge]) -> None:
+ cursor = self.conn.cursor()
+ try:
+ if edges:
+ cursor.execute(
+ f"""INSERT INTO {self.edges_table} (path, edges)
+ VALUES (?, ?)
+ ON CONFLICT(path) DO UPDATE SET edges = excluded.edges""",
+ (path, json.dumps(
+ [e.model_dump(exclude_none=True) for e in edges],
+ ensure_ascii=False,
+ )),
+ )
+ else:
+ cursor.execute(f"DELETE FROM {self.edges_table} WHERE path = ?", (path,))
+ self.conn.commit()
+ except Exception as e:
+ self.conn.rollback()
+ self.logger.error(f"Failed to upsert edges for {path}: {e}")
+ raise
+ finally:
+ cursor.close()
+
+ async def _persist_delete_edges(self, path: str) -> None:
+ cursor = self.conn.cursor()
+ try:
+ cursor.execute(f"DELETE FROM {self.edges_table} WHERE path = ?", (path,))
+ self.conn.commit()
+ except Exception as e:
+ self.conn.rollback()
+ self.logger.error(f"Failed to delete edges for {path}: {e}")
+ raise
+ finally:
+ cursor.close()
+
+ def _iter_persisted_edges(self) -> Iterable[tuple[str, list[FileEdge]]]:
+ if self.conn is None:
+ return []
+ cursor = self.conn.cursor()
+ try:
+ cursor.execute(f"SELECT path, edges FROM {self.edges_table}")
+ out: list[tuple[str, list[FileEdge]]] = []
+ for path, edges_json in cursor.fetchall():
+ try:
+ raws = json.loads(edges_json) if edges_json else []
+ out.append((path, [FileEdge.model_validate(e) for e in raws]))
+ except Exception as e:
+ self.logger.warning(f"Bad edge row for {path}: {e}")
+ return out
+ finally:
+ cursor.close()
diff --git a/reme2/component/file_watcher/base_file_watcher.py b/reme2/component/file_watcher/base_file_watcher.py
index d0740b79..544a8af4 100644
--- a/reme2/component/file_watcher/base_file_watcher.py
+++ b/reme2/component/file_watcher/base_file_watcher.py
@@ -1,82 +1,95 @@
-"""Base file watcher with watchfiles integration."""
+"""Base file watcher with watchfiles integration.
+
+Per change → single-step pipeline:
+ added/modified → existing = file_store.get_chunks(path)
+ parsed = parser.parse(path, existing_chunks=existing)
+ file_store.upsert_parsed(parsed)
+ deleted → file_store.delete_chunks + delete_file_meta
+
+The parser owns: chunking, edge extraction, and embedding (with hash-diff
+cache via the `existing_chunks` argument). The file_store owns: persistence
+of meta + edges + chunks (atomic-from-the-caller's-POV via `upsert_parsed`).
+The watcher owns: scheduling, cancel-on-modify, retry/timeout, and
+startup recovery (re-parsing files whose mtime drifted while offline).
+"""
import asyncio
+import time
from pathlib import Path
from watchfiles import Change, awatch
from ..base_component import BaseComponent
-from ..chunk_store import BaseChunkStore
from ..file_parser import BaseFileParser
+from ..file_store import BaseFileStore
from ...enumeration import ComponentEnum
-from ...schema.file_graph import FileGraph
class BaseFileWatcher(BaseComponent):
- """Abstract base class for file watchers.
-
- Provides file monitoring with:
- - watchfiles integration for efficient change detection
- - Parser-based file filtering
- - Auto-restart on failure
- - Optional index rebuild on start
- """
+ """Watches a directory and feeds the file_store via single-step parse+upsert."""
component_type = ComponentEnum.FILE_WATCHER
+ _META_DIR = ".reme"
+
def __init__(
- self,
- watch_path: str,
- recursive: bool = False,
- debounce: int = 2000,
- chunk_tokens: int = 400,
- chunk_overlap: int = 80,
- chunk_store: str = "default",
- default_parser: str | None = None,
- rebuild_index_on_start: bool = False,
- poll_delay_ms: int = 2000,
- **kwargs,
+ self,
+ watch_path: str,
+ recursive: bool = False,
+ debounce: int = 2000,
+ file_store: str = "default",
+ default_parser: str | None = None,
+ rebuild_index_on_start: bool = False,
+ poll_delay_ms: int = 2000,
+ parse_max_attempts: int = 3,
+ parse_retry_backoff: float = 2.0,
+ parse_task_timeout: float = 300.0,
+ **kwargs,
):
super().__init__(**kwargs)
- self._chunk_store_name: str = chunk_store
+ self._file_store_name: str = file_store
self._default_parser_name: str | None = default_parser
- self.chunk_store: BaseChunkStore | None = None
+ self.file_store: BaseFileStore | None = None
self._suffix_to_parser: dict[str, BaseFileParser] = {}
self._default_parser: BaseFileParser | None = None
self.watch_path: str = watch_path
self.recursive: bool = recursive
self.debounce: int = debounce
- self.chunk_tokens: int = chunk_tokens
- self.chunk_overlap: int = chunk_overlap
self.rebuild_index_on_start: bool = rebuild_index_on_start
self.poll_delay_ms: int = poll_delay_ms
- self.file_graph: FileGraph = FileGraph()
self._stop_event = asyncio.Event()
self._watch_task: asyncio.Task | None = None
- _META_DIR = ".reme"
+ # Parse pipeline configuration.
+ self._parse_max_attempts: int = max(1, int(parse_max_attempts))
+ self._parse_retry_backoff: float = float(parse_retry_backoff)
+ self._parse_task_timeout: float = float(parse_task_timeout)
+ self._tasks: dict[str, asyncio.Task] = {}
+ self._mgmt_lock: asyncio.Lock | None = None
+ self._run_lock: asyncio.Lock | None = None
+ self._last_failure: dict[str, float] = {}
@property
def _meta_path(self) -> Path:
- return Path(self.watch_path) / self._META_DIR
+ return (Path(self.watch_path) / self._META_DIR).resolve()
- @property
- def _graph_path(self) -> Path:
- return self._meta_path / "file_graph.json"
+ # -- Lifecycle ----------------------------------------------------------
async def _start(self):
- """Resolve chunk_store, load or build file_graph, and start watching."""
- if self._chunk_store_name:
+ """Resolve file_store + parsers, sync any disk drift, start watch loop."""
+ if self._file_store_name:
assert self.app_context is not None, "app_context must be provided"
- stores = self.app_context.components.get(ComponentEnum.CHUNK_STORE, {})
- if self._chunk_store_name not in stores:
- raise ValueError(f"Chunk store '{self._chunk_store_name}' not found.")
- store = stores[self._chunk_store_name]
- if not isinstance(store, BaseChunkStore):
- raise TypeError(f"Expected BaseChunkStore, got {type(store).__name__}")
- self.chunk_store = store
+ stores = self.app_context.components.get(ComponentEnum.FILE_STORE, {})
+ if self._file_store_name not in stores:
+ raise ValueError(f"File store '{self._file_store_name}' not found.")
+ store = stores[self._file_store_name]
+ if not isinstance(store, BaseFileStore):
+ raise TypeError(f"Expected BaseFileStore, got {type(store).__name__}")
+ self.file_store = store
+ # Vault root drives explicit-path wikilink resolution.
+ self.file_store.set_vault_root(self.watch_path)
parsers = self.app_context.components.get(ComponentEnum.FILE_PARSER, {})
for parser in parsers.values():
@@ -93,28 +106,16 @@ class BaseFileWatcher(BaseComponent):
self.logger.warning("No file parsers registered")
async def _initialize_and_watch():
- await self._load_or_build_graph()
+ await self._initial_sync_and_recovery()
await self._watch_loop()
+ self._mgmt_lock = asyncio.Lock()
+ self._run_lock = asyncio.Lock()
+ self._last_failure.clear()
self._stop_event.clear()
self._watch_task = asyncio.create_task(_initialize_and_watch())
self.logger.info(f"Started watching: {self.watch_path}")
- async def _load_or_build_graph(self) -> None:
- graph_path = self._graph_path
- if not self.rebuild_index_on_start and graph_path.exists():
- self.file_graph = FileGraph.load(graph_path)
- self.logger.info(f"Loaded file graph from {graph_path} ({len(self.file_graph)} nodes)")
- else:
- if self.rebuild_index_on_start:
- self.logger.info("Rebuild on start enabled, scanning to build")
- else:
- self.logger.info("No persisted file graph found, scanning to build")
- self.file_graph = FileGraph()
- await self._scan_existing_files()
- self.file_graph.save(graph_path)
- self.logger.info(f"Saved file graph to {graph_path} ({len(self.file_graph)} nodes)")
-
async def _close(self):
"""Stop watching and release resources."""
self._stop_event.set()
@@ -125,39 +126,94 @@ class BaseFileWatcher(BaseComponent):
except asyncio.CancelledError:
pass
+ # Cancel all in-flight parse tasks before tearing down state.
+ tasks = list(self._tasks.values())
+ for t in tasks:
+ if not t.done():
+ t.cancel()
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
+ self._tasks.clear()
+ self._last_failure.clear()
+
self._watch_task = None
self._stop_event.clear()
- self.chunk_store = None
+ self._mgmt_lock = None
+ self._run_lock = None
+ self.file_store = None
self._suffix_to_parser.clear()
self._default_parser = None
self.logger.info("Stopped watching")
- async def _scan_existing_files(self) -> None:
- if not self.chunk_store:
- return
+ # -- Startup sync -------------------------------------------------------
+
+ async def _initial_sync_and_recovery(self) -> None:
+ """At startup: file_store has its persisted state loaded already; we
+ diff against current disk for any changes that happened while we
+ were offline, then re-enqueue them through the parse pipeline."""
+ await self._sync_with_disk()
+
+ async def _sync_with_disk(self) -> bool:
+ """Diff cached graph (in file_store) vs disk state; reindex changes."""
+ if self.file_store is None:
+ return False
watch_path = Path(self.watch_path)
if not watch_path.exists():
self.logger.warning(f"Watch path does not exist: {watch_path}")
- return
+ return False
- existing_files: set[tuple[Change, str]] = set()
+ on_disk: dict[str, float] = {}
if watch_path.is_file():
- existing_files.add((Change.added, str(watch_path)))
+ on_disk[str(watch_path.resolve())] = watch_path.stat().st_mtime
elif watch_path.is_dir():
iterator = watch_path.rglob("*") if self.recursive else watch_path.iterdir()
- for file_path in iterator:
- if file_path.is_file() and self._watch_filter(str(file_path)):
- existing_files.add((Change.added, str(file_path)))
+ for fp in iterator:
+ if not fp.is_file():
+ continue
+ abs_path = str(fp.resolve())
+ if not self._watch_filter(abs_path):
+ continue
+ if not self._get_parser(abs_path):
+ continue
+ try:
+ on_disk[abs_path] = fp.stat().st_mtime
+ except OSError:
+ continue
- if existing_files:
- self.logger.info(f"Scanning {len(existing_files)} existing files")
- await self.on_changes(existing_files)
- else:
- self.logger.info("No existing files found")
+ cached_paths = set(self.file_store.nodes.keys())
+ disk_paths = set(on_disk.keys())
+
+ added = disk_paths - cached_paths
+ deleted = cached_paths - disk_paths
+ modified: set[str] = set()
+ for p in disk_paths & cached_paths:
+ cached_meta = self.file_store.get_file_meta(p)
+ if cached_meta is None or cached_meta.st_mtime != on_disk[p]:
+ modified.add(p)
+
+ unchanged = len(disk_paths & cached_paths) - len(modified)
+ if not (added or deleted or modified):
+ self.logger.info(f"Index up-to-date ({unchanged} files cached, 0 changes)")
+ return False
+
+ self.logger.info(
+ f"Incremental sync: +{len(added)} ~{len(modified)} -{len(deleted)} (unchanged {unchanged})",
+ )
+
+ changes: set[tuple[Change, str]] = set()
+ for p in deleted:
+ changes.add((Change.deleted, p))
+ for p in added:
+ changes.add((Change.added, p))
+ for p in modified:
+ changes.add((Change.modified, p))
+ await self.on_changes(changes)
+ return True
+
+ # -- Watch loop ---------------------------------------------------------
async def _interruptible_sleep(self, seconds: float):
- """Sleep that can be interrupted by stop_event."""
try:
await asyncio.wait_for(self._stop_event.wait(), timeout=seconds)
except asyncio.TimeoutError:
@@ -173,12 +229,12 @@ class BaseFileWatcher(BaseComponent):
try:
self.logger.info(f"Starting watch on: {self.watch_path}")
async for changes in awatch(
- self.watch_path,
- watch_filter=lambda _, p: self._watch_filter(p),
- recursive=self.recursive,
- debounce=self.debounce,
- poll_delay_ms=self.poll_delay_ms,
- stop_event=self._stop_event,
+ self.watch_path,
+ watch_filter=lambda _, p: self._watch_filter(p),
+ recursive=self.recursive,
+ debounce=self.debounce,
+ poll_delay_ms=self.poll_delay_ms,
+ stop_event=self._stop_event,
):
if self._stop_event.is_set():
break
@@ -195,22 +251,25 @@ class BaseFileWatcher(BaseComponent):
await self._interruptible_sleep(10)
def _watch_filter(self, path: str) -> bool:
- return self._meta_path not in Path(path).parents
+ resolved = Path(path).resolve()
+ if resolved == self._meta_path:
+ return False
+ return self._meta_path not in resolved.parents
def _get_parser(self, path: str) -> BaseFileParser | None:
suffix = Path(path).suffix.lower()
return self._suffix_to_parser.get(suffix, self._default_parser)
+ # -- Change dispatch ----------------------------------------------------
+
async def on_changes(self, changes: set[tuple[Change, str]]) -> None:
- if not self.chunk_store:
+ if self.file_store is None:
self.logger.warning("File store not initialized, skipping changes")
return
for change_type, path in changes:
try:
- if change_type == Change.added:
- await self._on_added(path)
- elif change_type == Change.modified:
+ if change_type in (Change.added, Change.modified):
await self._on_modified(path)
elif change_type == Change.deleted:
await self._on_deleted(path)
@@ -219,29 +278,115 @@ class BaseFileWatcher(BaseComponent):
except PermissionError:
self.logger.warning(f"Permission denied: {path}, skipping")
except Exception as e:
- self.logger.error(f"Error processing {path}: {e}", exc_info=True)
-
- async def _on_added(self, path: str) -> None:
- parser = self._get_parser(path)
- if not parser:
- self.logger.debug(f"No parser for {path}, skipping")
- return
- file_meta, chunks = await parser.parse(path)
- await self.chunk_store.upsert_chunks(path, chunks)
- self.file_graph.create(file_meta)
- self.logger.info(f"Added {path} ({len(chunks)} chunks)")
+ self.logger.opt(exception=True).error("Error processing {p}: {err}", p=path, err=str(e))
async def _on_modified(self, path: str) -> None:
+ if not Path(path).is_file():
+ return
parser = self._get_parser(path)
- if not parser:
+ if parser is None:
self.logger.debug(f"No parser for {path}, skipping")
return
- file_meta, chunks = await parser.parse(path)
- await self.chunk_store.upsert_chunks(path, chunks)
- self.file_graph.create(file_meta)
- self.logger.info(f"Modified {path} ({len(chunks)} chunks)")
+ await self._submit_parse_task(path, parser)
async def _on_deleted(self, path: str) -> None:
- await self.chunk_store.delete_chunks(path)
- self.file_graph.delete(path)
+ # Cancel any in-flight parse task FIRST so a delayed cancellation
+ # can't race the cleanup writes below.
+ assert self.file_store is not None, "_on_deleted requires file_store"
+ await self._cancel_parse_task(path)
+ await self.file_store.delete_chunks(path)
+ await self.file_store.delete_file_meta(path)
self.logger.info(f"Deleted {path}")
+
+ # -- Parse task pipeline ------------------------------------------------
+
+ async def _submit_parse_task(self, path: str, parser: BaseFileParser) -> None:
+ """Cancel any prior task + spawn a new one. Atomic under _mgmt_lock."""
+ assert self._mgmt_lock is not None, "_submit_parse_task before _start()"
+ async with self._mgmt_lock:
+ await self._cancel_locked(path)
+ self._tasks[path] = asyncio.create_task(
+ self._run_parse_task(path, parser),
+ )
+
+ async def _cancel_parse_task(self, path: str) -> None:
+ """Cancel + await any in-flight parse task for `path`."""
+ assert self._mgmt_lock is not None, "_cancel_parse_task before _start()"
+ async with self._mgmt_lock:
+ await self._cancel_locked(path)
+
+ async def _cancel_locked(self, path: str) -> None:
+ """Cancel + await the task for `path`. Caller holds `_mgmt_lock`."""
+ task = self._tasks.pop(path, None)
+ if task is not None and not task.done():
+ task.cancel()
+ try:
+ await task
+ except (asyncio.CancelledError, Exception):
+ pass
+
+ async def flush_parse_tasks(self) -> None:
+ """Wait until all parse tasks finish (loops because retries spawn new ones)."""
+ while self._tasks:
+ await asyncio.gather(*list(self._tasks.values()), return_exceptions=True)
+
+ def pending_parse_count(self) -> int:
+ return sum(1 for t in self._tasks.values() if not t.done())
+
+ def failed_parse_paths(self) -> dict[str, float]:
+ return dict(self._last_failure)
+
+ async def _run_parse_task(self, path: str, parser: BaseFileParser) -> None:
+ """One per-path parse task: get prior chunks → parse → upsert_parsed.
+
+ Hash diff happens inside the parser via `existing_chunks`: chunks
+ whose hash matches a stored chunk reuse the prior embedding; only
+ new-hash blocks hit the embedding API. The whole upsert is one
+ call to the file_store.
+ """
+ try:
+ for attempt in range(1, self._parse_max_attempts + 1):
+ try:
+ assert self.file_store is not None
+ existing = await self.file_store.get_chunks(path)
+ parsed = await asyncio.wait_for(
+ parser.parse(path, existing_chunks=existing),
+ timeout=self._parse_task_timeout,
+ )
+
+ assert self._run_lock is not None
+ async with self._run_lock:
+ await asyncio.wait_for(
+ self.file_store.upsert_parsed(parsed),
+ timeout=self._parse_task_timeout,
+ )
+ self._last_failure.pop(path, None)
+
+ self.logger.info(
+ f"indexed {path}: chunks={len(parsed.chunks)} edges={len(parsed.edges)}",
+ )
+ return
+ except asyncio.CancelledError:
+ raise
+ except FileNotFoundError:
+ self.logger.debug(f"parse task {path}: file gone, skipping")
+ self._last_failure.pop(path, None)
+ return
+ except Exception as e:
+ self._last_failure[path] = time.time()
+ if attempt < self._parse_max_attempts:
+ backoff = self._parse_retry_backoff * attempt
+ self.logger.warning(
+ f"parse task {path} attempt {attempt} failed "
+ f"({type(e).__name__}: {e}); retry in {backoff:.1f}s",
+ )
+ await asyncio.sleep(backoff)
+ continue
+ self.logger.error(
+ f"parse task {path} giving up after {attempt} attempts: "
+ f"{type(e).__name__}: {e}",
+ )
+ return
+ finally:
+ if self._tasks.get(path) is asyncio.current_task():
+ self._tasks.pop(path, None)
diff --git a/reme2/component/job/base_job.py b/reme2/component/job/base_job.py
index ea6766e3..177467cb 100644
--- a/reme2/component/job/base_job.py
+++ b/reme2/component/job/base_job.py
@@ -1,4 +1,5 @@
"""Base job component for sequential step execution."""
+
from typing import TYPE_CHECKING
from ..base_component import BaseComponent
@@ -18,11 +19,11 @@ class BaseJob(BaseComponent):
component_type = ComponentEnum.JOB
def __init__(
- self,
- description: str = "",
- parameters: dict | None = None,
- steps: list[ComponentConfig] | None = None,
- **kwargs,
+ self,
+ description: str = "",
+ parameters: dict | None = None,
+ steps: list[ComponentConfig] | None = None,
+ **kwargs,
):
super().__init__(**kwargs)
self.description = description
@@ -32,14 +33,14 @@ class BaseJob(BaseComponent):
async def _start(self) -> None:
assert self.app_context is not None, "app_context must be provided"
- for config in self.step_configs:
+ for raw in self.step_configs:
+ config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
if not config.backend:
raise ValueError(f"Step is missing the required 'backend' field")
step_cls = R.get(ComponentEnum.STEP, config.backend)
if not step_cls:
raise ValueError(
- f"Step references an unregistered backend '{config.backend}' "
- f"of type '{ComponentEnum.STEP}'",
+ f"Step references an unregistered backend '{config.backend}' " f"of type '{ComponentEnum.STEP}'",
)
params = config.model_dump()
params["app_context"] = self.app_context
diff --git a/reme2/component/prompt_handler.py b/reme2/component/prompt_handler.py
index 891ce242..575c82b1 100644
--- a/reme2/component/prompt_handler.py
+++ b/reme2/component/prompt_handler.py
@@ -21,9 +21,9 @@ class PromptHandler:
self.language: str = language.strip()
def load_prompt_by_file(
- self,
- prompt_file_path: str | Path | None = None,
- overwrite: bool = True,
+ self,
+ prompt_file_path: str | Path | None = None,
+ overwrite: bool = True,
) -> "PromptHandler":
"""Load prompts from a YAML or JSON file."""
if prompt_file_path is None:
diff --git a/reme2/component/runtime_response.py b/reme2/component/runtime_response.py
new file mode 100644
index 00000000..92b2732f
--- /dev/null
+++ b/reme2/component/runtime_response.py
@@ -0,0 +1,30 @@
+"""Helpers for serializing Step output onto `RuntimeContext.response`.
+
+Lives next to `runtime_context.py` because both are about the BaseStep
+interface — the response side, specifically. Used by every Step that
+returns a JSON-shaped payload (memory_*, sync, topic_create, the three
+memory services).
+
+Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP
+dependency into `reme2/memory/` services that legitimately need to
+serialize their results — the moved location breaks that cycle.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date, datetime
+
+
+def _to_jsonable(value):
+ if isinstance(value, (datetime, date)):
+ return value.isoformat()
+ if isinstance(value, dict):
+ return {k: _to_jsonable(v) for k, v in value.items()}
+ if isinstance(value, (list, tuple, set)):
+ return [_to_jsonable(v) for v in value]
+ return value
+
+
+def _set_answer(context, payload) -> None:
+ context.response.answer = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2)
diff --git a/reme2/component/service/base_service.py b/reme2/component/service/base_service.py
index e834ffc4..0f39820a 100644
--- a/reme2/component/service/base_service.py
+++ b/reme2/component/service/base_service.py
@@ -21,16 +21,13 @@ class BaseService(BaseComponent):
self.service = None
@abstractmethod
- def build_service(self, app: "Application") -> None:
- ...
+ def build_service(self, app: "Application") -> None: ...
@abstractmethod
- def add_job(self, job: BaseJob) -> None:
- ...
+ def add_job(self, job: BaseJob) -> None: ...
@abstractmethod
- def start_service(self, app: "Application") -> None:
- ...
+ def start_service(self, app: "Application") -> None: ...
def add_jobs(self, app: "Application") -> None:
for name, job in app.context.jobs.items():
diff --git a/reme2/component/service/http_service.py b/reme2/component/service/http_service.py
index 9b2d2762..5c4bef60 100644
--- a/reme2/component/service/http_service.py
+++ b/reme2/component/service/http_service.py
@@ -51,10 +51,10 @@ class HttpService(BaseService):
async def generate_stream() -> AsyncGenerator[bytes, None]:
async for chunk in execute_stream_task(
- stream_queue=stream_queue,
- task=task,
- task_name=job.name,
- output_format="bytes",
+ stream_queue=stream_queue,
+ task=task,
+ task_name=job.name,
+ output_format="bytes",
):
assert isinstance(chunk, bytes)
yield chunk
diff --git a/reme2/component/service/mcp_service.py b/reme2/component/service/mcp_service.py
index c662dee5..cabe3257 100644
--- a/reme2/component/service/mcp_service.py
+++ b/reme2/component/service/mcp_service.py
@@ -1,9 +1,20 @@
-"""Model Context Protocol (MCP) service implementation."""
+"""Model Context Protocol (MCP) service implementation.
+
+Optionally exposes an HTTP sidecar (off by default) on a local-only
+loopback port. The sidecar shares the running Application instance,
+so callers (lifecycle hooks, scripts) see the live file_store without
+booting a duplicate process.
+"""
+
+import asyncio
import json
import os
from contextlib import asynccontextmanager
+from pathlib import Path
from typing import TYPE_CHECKING
+import uvicorn
+from fastapi import FastAPI, Request as FastAPIRequest
from fastmcp import FastMCP
from fastmcp.server.server import Transport
from fastmcp.tools import FunctionTool
@@ -18,20 +29,54 @@ if TYPE_CHECKING:
from ..job import BaseJob
+def _build_sidecar_app(app: "Application") -> FastAPI:
+ """A minimal FastAPI app that exposes each registered job as POST /."""
+ api = FastAPI(title=f"{app.config.app_name}-sidecar")
+
+ @api.get("/health")
+ async def health():
+ return {"status": "ok", "jobs": list(app.context.jobs.keys())}
+
+ for job_name in list(app.context.jobs.keys()):
+ # bind job_name via default arg to avoid late-binding in the closure
+ async def endpoint(request: FastAPIRequest, _name: str = job_name):
+ try:
+ payload = await request.json()
+ except Exception:
+ payload = {}
+ response = await app.run_job(_name, **(payload or {}))
+ return {"answer": response.answer, "success": response.success}
+
+ api.post(f"/{job_name}")(endpoint)
+
+ return api
+
+
@R.register("mcp")
class MCPService(BaseService):
"""Expose jobs as Model Context Protocol (MCP) tools."""
def __init__(
- self,
- transport: Transport = "sse",
- host: str = REME_DEFAULT_HOST,
- port: int = REME_DEFAULT_PORT,
- **kwargs):
+ self,
+ transport: Transport = "sse",
+ host: str = REME_DEFAULT_HOST,
+ port: int = REME_DEFAULT_PORT,
+ sidecar_http: bool = False,
+ sidecar_http_host: str = "127.0.0.1",
+ sidecar_http_port: int = 8765,
+ sidecar_info_path: str = "",
+ **kwargs,
+ ):
super().__init__(**kwargs)
self.transport: Transport = transport
self.host: str = host
self.port: int = port
+ self.sidecar_http: bool = sidecar_http
+ self.sidecar_http_host: str = sidecar_http_host
+ self.sidecar_http_port: int = sidecar_http_port
+ # Optional file path where {host, port} are dropped at startup so that
+ # external scripts (lifecycle hooks) can discover the sidecar URL.
+ self.sidecar_info_path: str = sidecar_info_path
def build_service(self, app: "Application") -> None:
@@ -41,11 +86,59 @@ class MCPService(BaseService):
service_info = json.dumps({"host": self.host, "port": self.port})
os.environ[REME_SERVICE_INFO] = service_info
self.logger.info(f"ReMe MCP Service started: {REME_SERVICE_INFO}={service_info}")
- yield
- await app.close()
+
+ sidecar_task: asyncio.Task | None = None
+ if self.sidecar_http:
+ sidecar_task = asyncio.create_task(self._serve_sidecar(app))
+
+ try:
+ yield
+ finally:
+ if sidecar_task is not None:
+ sidecar_task.cancel()
+ try:
+ await sidecar_task
+ except (asyncio.CancelledError, Exception):
+ pass
+ self._cleanup_sidecar_info()
+ await app.close()
self.service = FastMCP(name=app.config.app_name, lifespan=lifespan)
+ async def _serve_sidecar(self, app: "Application") -> None:
+ sidecar_app = _build_sidecar_app(app)
+ config = uvicorn.Config(
+ sidecar_app,
+ host=self.sidecar_http_host,
+ port=self.sidecar_http_port,
+ log_level="warning",
+ access_log=False,
+ )
+ server = uvicorn.Server(config)
+ self._publish_sidecar_info()
+ self.logger.info(
+ f"MCP HTTP sidecar listening on " f"http://{self.sidecar_http_host}:{self.sidecar_http_port}",
+ )
+ await server.serve()
+
+ def _publish_sidecar_info(self) -> None:
+ if not self.sidecar_info_path:
+ return
+ info_path = Path(self.sidecar_info_path)
+ info_path.parent.mkdir(parents=True, exist_ok=True)
+ info_path.write_text(
+ json.dumps({"host": self.sidecar_http_host, "port": self.sidecar_http_port}),
+ encoding="utf-8",
+ )
+
+ def _cleanup_sidecar_info(self) -> None:
+ if not self.sidecar_info_path:
+ return
+ try:
+ Path(self.sidecar_info_path).unlink(missing_ok=True)
+ except Exception:
+ pass
+
def add_job(self, job: "BaseJob") -> None:
if isinstance(job, StreamJob):
return
diff --git a/reme2/config/config_parser.py b/reme2/config/config_parser.py
index c4a0d7d5..72f8cb07 100644
--- a/reme2/config/config_parser.py
+++ b/reme2/config/config_parser.py
@@ -1,6 +1,8 @@
"""Parser for YAML config with CLI argument overrides."""
import json
+import os
+import re
from pathlib import Path
from typing import Any
@@ -9,6 +11,32 @@ import yaml
# Config files are looked up relative to this module's directory
_CONFIG_DIR = Path(__file__).parent
_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
+_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
+
+
+def _expand_env_vars(value: Any) -> Any:
+ """Recursively expand `${VAR}` / `${VAR:-default}` placeholders in strings.
+
+ Lets vault.yaml reference secrets / per-host settings without baking
+ them into the file. Unset vars without a default raise ValueError so
+ typos don't silently produce empty connection strings.
+ """
+ if isinstance(value, str):
+ def repl(m: re.Match) -> str:
+ name = m.group(1)
+ default = m.group(2)
+ v = os.environ.get(name)
+ if v is None:
+ if default is not None:
+ return default
+ raise ValueError(f"Config references undefined env var: {name}")
+ return v
+ return _ENV_VAR_RE.sub(repl, value)
+ if isinstance(value, dict):
+ return {k: _expand_env_vars(v) for k, v in value.items()}
+ if isinstance(value, list):
+ return [_expand_env_vars(v) for v in value]
+ return value
# Pre-scan config directory: maps basename(without ext) -> Path
@@ -91,14 +119,15 @@ def _load_yaml(name_or_path: str, encoding: str = "utf-8") -> dict:
def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
- """Read YAML or JSON file based on extension."""
+ """Read YAML or JSON file based on extension. Expands ${ENV_VAR}."""
with path.open(encoding=encoding) as f:
if path.suffix == ".json":
result = json.load(f)
- return result if result is not None else {}
else:
result = yaml.safe_load(f)
- return result if result is not None else {}
+ if result is None:
+ return {}
+ return _expand_env_vars(result)
def _deep_merge(base: dict, update: dict) -> dict:
diff --git a/reme2/config/curated.yaml b/reme2/config/curated.yaml
new file mode 100644
index 00000000..37ac2343
--- /dev/null
+++ b/reme2/config/curated.yaml
@@ -0,0 +1,220 @@
+app_name: reme-curated
+enable_logo: false
+log_to_console: true
+log_to_file: false
+
+# Curated profile: three MCP tools surfaced —
+# `query` — graph-aware hybrid retrieval
+# `sync` — hot-path event sync (idempotent upsert per (date, name))
+# `ingest` — LLM-driven distillation over existing files (cold path)
+#
+# The agent calls `sync` continuously through the task — picking a
+# stable `name` per logical thread so each call extends the same
+# event folder rather than fragmenting. At task completion (or
+# PreCompact / SessionEnd) it calls `ingest` once to distill the
+# active events into topic-level cognition.
+#
+# For full direct control over every memory_* primitive see ./full.yaml.
+
+service:
+ backend: mcp
+ transport: stdio
+ sidecar_http: true
+ sidecar_http_host: "127.0.0.1"
+ sidecar_http_port: 8765
+ sidecar_info_path: "./vault/.reme/sidecar.json"
+
+jobs:
+ - backend: base
+ name: query
+ description: |
+ Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink
+ BFS fusion. Use for "what do I know about X" / "did I work on Y" /
+ "what's connected to [[Z]]". Anchor mode: include `[[Target]]` in
+ the query to seed BFS at that file. Topic-rooted mode: pass `seeds`
+ explicitly. Returns chunks ranked by combined relevance + graph
+ proximity, each tagged with `graph_hop`.
+ parameters:
+ type: object
+ properties:
+ query: { type: string }
+ max_results: { type: integer, default: 5 }
+ min_score: { type: number, default: 0.0 }
+ graph_depth:
+ type: integer
+ default: 1
+ description: "BFS hops from seeds. 1 covers immediate neighbors."
+ seeds:
+ type: array
+ items: { type: string }
+ description: "Explicit seed paths (topic-rooted mode)."
+ paths: { type: array, items: { type: string } }
+ tags: { type: array, items: { type: string } }
+ exclude_paths: { type: array, items: { type: string } }
+ steps:
+ - backend: memory_graph_search
+
+ - backend: base
+ name: sync
+ description: |
+ Hot-path event sync: idempotent upsert of an event FOLDER under
+ `events/{date}/{name}/`. The folder contains the index `{name}.md`
+ (Event schema, frontmatter + narrative + Materials footer) plus
+ any raw materials you pass — conversation snippets, tool outputs,
+ data dumps. The watcher indexes everything inside.
+
+ CONTINUITY MODEL: pick a stable `name` per logical thread and
+ call `sync` repeatedly through the task. Each call extends the
+ same folder:
+ * new `content` → appended under a `## Update — {iso}` section
+ * new `materials` → siblings (auto-suffix on filename collision)
+ * `topics` + `tags` merged (union) into frontmatter
+ * Materials footer regenerated to list every artifact in the folder
+
+ First call (folder doesn't exist) → CREATE; subsequent calls with
+ the same `name` while the event is `status: active` → APPEND.
+ If the event is `status: distilled` / `archived`, `sync` REFUSES
+ and returns a `suggested_name` so you start a fresh thread instead
+ of mutating prior cognition.
+
+ Zero LLM cost. Call CONTINUOUSLY through a task as facts land,
+ and especially at PreCompact to dump verbose raw text into
+ `materials` before context truncation loses it. The folder is
+ the unit `ingest` later reads from.
+ parameters:
+ type: object
+ required: [name]
+ properties:
+ name:
+ type: string
+ description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
+ description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" }
+ content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." }
+ topics:
+ type: array
+ items: { type: string }
+ description: "related topic wikilinks. Unioned into frontmatter on append."
+ tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." }
+ materials:
+ type: array
+ description: "Raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision with an existing artifact auto-suffixes (foo.txt → foo-2.txt)."
+ items:
+ type: object
+ required: [filename, content]
+ properties:
+ filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" }
+ content: { type: string }
+ on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" }
+ origin_session_id: { type: string, description: "set on initial create only" }
+ steps:
+ - backend: sync
+
+ - backend: base
+ name: ingest
+ description: |
+ Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only:
+ task completion / SessionEnd / when the agent decides the working
+ set is ready. Not a per-turn tool.
+
+ The agent hands off the working set in two interchangeable forms
+ (use both as appropriate):
+ * `content` — inline material to distill: a hint / summary, or
+ raw text the agent is feeding directly.
+ * `related_paths` — paths the agent points at: event folder
+ indexes (Ingestor follows `## Materials` to read each
+ artifact), individual material files, or candidate topics.
+
+ The Ingestor reads the working set + linked topics, decides which
+ existing topics to update / create, and flips each distilled
+ event's status to "distilled". Returns an audit trail.
+
+ NOT for raw event logging — that's `sync`'s job.
+ parameters:
+ type: object
+ required: [content]
+ properties:
+ content:
+ type: string
+ description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set."
+ hint:
+ type: string
+ description: "Caller guidance about target / intent."
+ target_path:
+ type: string
+ description: "Optional suggested path; required for the no-LLM degraded path."
+ metadata:
+ type: object
+ description: "Suggested frontmatter for any new topic."
+ related_paths:
+ type: array
+ items: { type: string }
+ description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics."
+ steps:
+ - backend: ingestor
+
+components:
+ # Ingestor LLM (opt-in). Without this the Ingestor degrades to a
+ # direct create from explicit `target_path`; edits/renames/deletes
+ # require the LLM. Uncomment + provide LLM_API_KEY to enable.
+ #
+ # as_llm:
+ # default:
+ # backend: openai
+ # model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
+ # api_key: ${LLM_API_KEY}
+ # client_kwargs:
+ # base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
+ # stream: false
+ #
+ # as_llm_formatter:
+ # default:
+ # backend: openai
+
+ as_token_counter:
+ default:
+ backend: estimated
+
+ # Embedding is opt-in: leave embedding_model="" on file_store to run
+ # keyword-only; uncomment + flip to "default" to enable hybrid search.
+ #
+ # embedding_model:
+ # default:
+ # backend: openai
+ # model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
+ # dimensions: 1536
+ # pass_dimensions: false
+ # enable_cache: true
+ # max_batch_size: 10
+ # max_cache_size: 2000
+ # max_input_length: 8192
+
+ edge_extractor:
+ default:
+ backend: regex
+
+ file_parser:
+ md:
+ backend: md
+ edge_extractor: default
+ default:
+ backend: text
+
+ file_store:
+ default:
+ backend: local
+ embedding_model: ""
+ store_name: "reme"
+ db_path: "./vault/.reme"
+ fts_enabled: true
+
+ file_watcher:
+ default:
+ backend: full
+ file_store: default
+ default_parser: md
+ watch_path: "./vault"
+ recursive: true
+
+ # Retriever (`hybrid`) is a Step, not a pre-instantiated component —
+ # the `query` shell builds it on demand. Tune defaults by attaching
+ # knobs to the `memory_graph_search` step under the `query` job above.
diff --git a/reme2/config/full.yaml b/reme2/config/full.yaml
new file mode 100644
index 00000000..7630371e
--- /dev/null
+++ b/reme2/config/full.yaml
@@ -0,0 +1,409 @@
+app_name: reme-full
+enable_logo: false
+log_to_console: true
+log_to_file: false
+
+# Full-exposure profile: every memory_* read + write primitive + the
+# typed `topic_create` + the LLM-driven `ingest` are surfaced as MCP
+# tools. The agent picks whatever it needs — finest granularity, no
+# opinion enforced beyond the wikilink-uniqueness gate.
+#
+# Use when you want the agent to manage memory directly with full
+# control. For an opinionated minimal surface, see ./curated.yaml
+# (only `query` + `ingest`).
+
+service:
+ backend: mcp
+ transport: stdio
+ sidecar_http: true
+ sidecar_http_host: "127.0.0.1"
+ sidecar_http_port: 8765
+ sidecar_info_path: "./vault/.reme/sidecar.json"
+
+jobs:
+ # -- Write entry points ------------------------------------------------
+
+ - backend: base
+ name: sync
+ description: |
+ Hot-path event sync: idempotent upsert of an event FOLDER under
+ `events/{date}/{name}/`. The folder contains the index `{name}.md`
+ (Event schema, frontmatter + narrative + Materials footer) plus
+ any raw materials you pass — conversation snippets, tool outputs,
+ data dumps. The watcher indexes everything inside.
+
+ CONTINUITY MODEL: pick a stable `name` per logical thread and
+ call `sync` repeatedly through the task. Each call extends the
+ same folder:
+ * new `content` → appended under a `## Update — {iso}` section
+ * new `materials` → siblings (auto-suffix on filename collision)
+ * `topics` + `tags` merged (union) into frontmatter
+ * Materials footer regenerated to list every artifact in the folder
+
+ First call (folder doesn't exist) → CREATE; subsequent calls with
+ the same `name` while the event is `status: active` → APPEND.
+ If the event is `status: distilled` / `archived`, `sync` REFUSES
+ and returns a `suggested_name` so you start a fresh thread
+ instead of mutating prior cognition.
+
+ Zero LLM cost. Call CONTINUOUSLY through a task as facts land,
+ and especially at PreCompact to dump verbose raw text into
+ `materials` before context truncation loses it. The folder is
+ the unit `ingest` later reads from.
+ parameters:
+ type: object
+ required: [name]
+ properties:
+ name:
+ type: string
+ description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
+ description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" }
+ content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." }
+ topics:
+ type: array
+ items: { type: string }
+ description: "related topic wikilinks ('[[X]]' or '[[topics/X/X]]'). Unioned into frontmatter on append."
+ tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." }
+ materials:
+ type: array
+ description: "Raw artifacts written as siblings of the index inside the event folder. Filenames must be safe (letters/digits/dot/underscore/dash). On filename collision with an existing artifact, auto-suffixes (foo.txt → foo-2.txt)."
+ items:
+ type: object
+ required: [filename, content]
+ properties:
+ filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt', 'snapshot.json'" }
+ content: { type: string }
+ on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" }
+ origin_session_id: { type: string, description: "set on initial create only" }
+ steps:
+ - backend: sync
+
+ - backend: base
+ name: ingest
+ description: |
+ Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only:
+ task completion / SessionEnd / when the agent decides the working
+ set is ready. Not a per-turn tool.
+
+ The agent hands off the working set in two interchangeable forms
+ (use both as appropriate):
+ * `content` — inline material to distill: a hint / summary, or
+ raw text the agent is feeding directly.
+ * `related_paths` — paths the agent points at: event folder
+ indexes (the Ingestor will follow `## Materials` and read
+ each artifact), individual material files, or candidate
+ topics flagged for update.
+
+ The Ingestor reads the working set + linked topics, decides which
+ existing topics to update / create, and flips each distilled
+ event's status to "distilled".
+
+ NOT for raw event logging — use `sync` for that.
+ parameters:
+ type: object
+ required: [content]
+ properties:
+ content:
+ type: string
+ description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set."
+ hint: { type: string, description: "Caller guidance about target / intent." }
+ target_path: { type: string, description: "Optional suggested path; required for the no-LLM degraded path." }
+ metadata: { type: object, description: "Suggested frontmatter for any new topic." }
+ related_paths:
+ type: array
+ items: { type: string }
+ description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor will read materials from `## Materials`), individual material files, or candidate topics."
+ steps:
+ - backend: ingestor
+
+ - backend: base
+ name: topic_create
+ description: |
+ Create a typed topic at `topics/{folder}/{name}.md` with hard
+ schema validation (judgment categories require `confidence`) and
+ wikilink-uniqueness enforcement.
+ parameters:
+ type: object
+ required: [folder, name, category]
+ properties:
+ folder: { type: string }
+ name: { type: string }
+ category:
+ type: string
+ enum: [company, sector, concept, method, tool, profile, thesis, model, questions, fundamentals]
+ description: { type: string }
+ content: { type: string }
+ confidence: { type: string, enum: ["⏳", "✅", "❌"] }
+ market: { type: string }
+ ticker: { type: string }
+ tags: { type: array, items: { type: string } }
+ steps:
+ - backend: topic_create
+
+ # -- Read tools --------------------------------------------------------
+
+ - backend: base
+ name: memory_search
+ description: "Hybrid (vector + keyword) search over chunks."
+ parameters:
+ type: object
+ required: [query]
+ properties:
+ query: { type: string }
+ max_results: { type: integer, default: 5 }
+ min_score: { type: number, default: 0.1 }
+ paths: { type: array, items: { type: string } }
+ tags: { type: array, items: { type: string } }
+ exclude_paths: { type: array, items: { type: string } }
+ steps:
+ - backend: memory_search
+
+ - backend: base
+ name: memory_graph_search
+ description: |
+ Three-way fusion search: vector + keyword + graph (BFS over wikilinks).
+ Pulls in chunks reachable through linked topics/events that pure
+ relevance search would miss.
+ parameters:
+ type: object
+ properties:
+ query: { type: string }
+ max_results: { type: integer, default: 5 }
+ min_score: { type: number, default: 0.0 }
+ graph_depth: { type: integer, default: 1 }
+ seeds: { type: array, items: { type: string } }
+ paths: { type: array, items: { type: string } }
+ tags: { type: array, items: { type: string } }
+ exclude_paths: { type: array, items: { type: string } }
+ steps:
+ - backend: memory_graph_search
+
+ - backend: base
+ name: memory_get
+ description: "Read frontmatter + body of a single file."
+ parameters:
+ type: object
+ required: [path]
+ properties:
+ path: { type: string }
+ include_chunks: { type: boolean, default: false }
+ steps:
+ - backend: memory_get
+
+ - backend: base
+ name: memory_list
+ description: |
+ List indexed files filtered by frontmatter exact-match, tags, or
+ path prefix. Returns {items: [{path, metadata}], count}.
+ parameters:
+ type: object
+ properties:
+ metadata: { type: object }
+ tags: { type: array, items: { type: string } }
+ path_prefix: { type: string }
+ limit: { type: integer, default: 100 }
+ steps:
+ - backend: memory_list
+
+ - backend: base
+ name: memory_backlinks
+ description: "Files linking TO the given path (with edge predicates)."
+ parameters:
+ type: object
+ required: [path]
+ properties:
+ path: { type: string }
+ steps:
+ - backend: memory_backlinks
+
+ - backend: base
+ name: memory_links
+ description: "Files the given path links to (resolved, with edge predicates)."
+ parameters:
+ type: object
+ required: [path]
+ properties:
+ path: { type: string }
+ steps:
+ - backend: memory_links
+
+ - backend: base
+ name: memory_resolve_wikilink
+ description: |
+ Resolve a `[[target]]` wikilink to a vault path. Stem-form (`X`)
+ consults the file_store's stem index; path-form (`a/b` or `a/b.md`)
+ is anchored at the vault root.
+ parameters:
+ type: object
+ required: [wikilink]
+ properties:
+ wikilink: { type: string }
+ steps:
+ - backend: memory_resolve_wikilink
+
+ - backend: base
+ name: memory_count_tokens
+ description: "Estimate token count for a file body or raw text."
+ parameters:
+ type: object
+ properties:
+ path: { type: string }
+ text: { type: string }
+ steps:
+ - backend: memory_count_tokens
+
+ # -- Write primitives (raw building blocks) ----------------------------
+
+ - backend: base
+ name: memory_create
+ description: |
+ Create a new file (raw primitive — no LLM reasoning). Prefer
+ `ingest` when you want an LLM curator. Wikilink-uniqueness gate
+ runs unless `force=true`.
+ parameters:
+ type: object
+ required: [path]
+ properties:
+ path: { type: string }
+ metadata: { type: object }
+ content: { type: string }
+ overwrite: { type: boolean, default: false }
+ force: { type: boolean, default: false }
+ steps:
+ - backend: memory_create
+
+ - backend: base
+ name: memory_update
+ description: |
+ Edit-style content update: replace `old_string` with `new_string`
+ in the file body. For frontmatter changes use `memory_property_update`.
+ parameters:
+ type: object
+ required: [path, old_string, new_string]
+ properties:
+ path: { type: string }
+ old_string: { type: string }
+ new_string: { type: string }
+ replace_all: { type: boolean, default: false }
+ steps:
+ - backend: memory_update
+
+ - backend: base
+ name: memory_property_update
+ description: "Update a single YAML frontmatter key. value=null deletes the key."
+ parameters:
+ type: object
+ required: [path, key]
+ properties:
+ path: { type: string }
+ key: { type: string }
+ value: {}
+ steps:
+ - backend: memory_property_update
+
+ - backend: base
+ name: memory_rename
+ description: |
+ Rename a file and rewrite all incoming `[[wikilink]]` references
+ across the vault. Refuses on destination conflict or stem ambiguity.
+ parameters:
+ type: object
+ required: [old_path, new_path]
+ properties:
+ old_path: { type: string }
+ new_path: { type: string }
+ steps:
+ - backend: memory_rename
+
+ - backend: base
+ name: memory_delete
+ description: "Delete a file."
+ parameters:
+ type: object
+ required: [path]
+ properties:
+ path: { type: string }
+ steps:
+ - backend: memory_delete
+
+ - backend: base
+ name: memory_archive
+ description: |
+ Archive a file: flip `status: archived` then move under
+ `//`.
+ parameters:
+ type: object
+ required: [path]
+ properties:
+ path: { type: string }
+ archive_dir: { type: string, default: "Archive" }
+ steps:
+ - backend: memory_archive
+
+components:
+ # Ingestor LLM (opt-in). Without this the Ingestor degrades to a
+ # direct create from explicit `target_path`; edits/renames/deletes
+ # require the LLM. Uncomment + provide LLM_API_KEY to enable.
+ #
+ # as_llm:
+ # default:
+ # backend: openai
+ # model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
+ # api_key: ${LLM_API_KEY}
+ # client_kwargs:
+ # base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
+ # stream: false
+ #
+ # as_llm_formatter:
+ # default:
+ # backend: openai
+
+ as_token_counter:
+ default:
+ backend: estimated
+
+ # Embedding is opt-in: leave embedding_model="" on file_store to run
+ # keyword-only; uncomment + flip to "default" to enable hybrid search.
+ #
+ # embedding_model:
+ # default:
+ # backend: openai
+ # model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
+ # dimensions: 1536
+ # pass_dimensions: false
+ # enable_cache: true
+ # max_batch_size: 10
+ # max_cache_size: 2000
+ # max_input_length: 8192
+
+ edge_extractor:
+ default:
+ backend: regex
+
+ file_parser:
+ md:
+ backend: md
+ edge_extractor: default
+ default:
+ backend: text
+
+ file_store:
+ default:
+ backend: local
+ embedding_model: ""
+ store_name: "reme"
+ db_path: "./vault/.reme"
+ fts_enabled: true
+
+ file_watcher:
+ default:
+ backend: full
+ file_store: default
+ default_parser: md
+ watch_path: "./vault"
+ recursive: true
+
+ # Retriever (`hybrid`) is a Step, not a pre-instantiated component —
+ # the `memory_search` / `memory_graph_search` shells build it on
+ # demand. To tune defaults, pass knobs (`vector_weight`, `graph_*`,
+ # …) on the step config under each job below.
diff --git a/reme2/config/paw.yaml b/reme2/config/paw.yaml
index 999ac9ed..efc994f9 100644
--- a/reme2/config/paw.yaml
+++ b/reme2/config/paw.yaml
@@ -42,13 +42,28 @@ components:
max_cache_size: 2000
max_input_length: 8192
+ edge_extractor:
+ default:
+ backend: regex
+ llm:
+ backend: llm
+ as_llm: default
+ as_llm_formatter: default
+ file_store: default
+ max_input_chars: 8000
+ min_confidence: 0.5
+ max_iters: 8
+
file_parser:
md:
backend: md
+ edge_extractor: default
+ embedding_model: default
default:
- backend: default
+ backend: text
+ embedding_model: default
- chunk_store:
+ file_store:
default:
backend: local
embedding_model: default
@@ -58,7 +73,7 @@ components:
file_watcher:
default:
backend: full
- chunk_store: default
+ file_store: default
default_parser: default
watch_paths: [ "./test_data" ]
recursive: true
diff --git a/reme2/config/smoke_test.py b/reme2/config/smoke_test.py
new file mode 100644
index 00000000..b874f2c2
--- /dev/null
+++ b/reme2/config/smoke_test.py
@@ -0,0 +1,308 @@
+"""Smoke-test reme2/config/full.yaml + reme2/config/curated.yaml.
+
+Boots Application with each profile against a temporary vault, lists
+the registered jobs, then calls a representative subset to confirm
+end-to-end wiring (parser → file_store → memory_io / ingest /
+memory_graph_search). The Ingestor degrades gracefully when no LLM is
+configured — we exercise the degraded path so the test is hermetic.
+
+Run:
+ python reme2/config/smoke_test.py
+"""
+
+import asyncio
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(_REPO_ROOT))
+
+# Eager-import side-effect modules so all @R.register() decorators run.
+import reme2 # noqa: E402,F401
+import reme2.mcp.steps.memory_io # noqa: E402,F401
+import reme2.mcp.steps.memory_retriever # noqa: E402,F401
+import reme2.memory.ingestor # noqa: E402,F401
+import reme2.memory.summarizer # noqa: E402,F401
+import reme2.memory.maintainer # noqa: E402,F401
+import reme2.mcp.steps # noqa: E402,F401
+
+from reme2.application import Application # noqa: E402
+from reme2.config import parse_args # noqa: E402
+
+CONFIG_DIR = Path(__file__).resolve().parent
+PROFILES = {
+ "full": CONFIG_DIR / "full.yaml",
+ "curated": CONFIG_DIR / "curated.yaml",
+}
+
+
+def _seed_vault(vault: Path) -> None:
+ """Drop a few markdown files so reads return something."""
+ (vault / "topics" / "Alice").mkdir(parents=True, exist_ok=True)
+ (vault / "topics" / "Alice" / "Alice.md").write_text(
+ "---\ntitle: Alice\ncategory: profile\ntags: [person]\n---\n"
+ "# Alice\n\nAlice works on [[Project X]] with [[Bob]].\n"
+ "[author:: [[Alice]]]\n",
+ encoding="utf-8",
+ )
+ (vault / "topics" / "Bob").mkdir(parents=True, exist_ok=True)
+ (vault / "topics" / "Bob" / "Bob.md").write_text(
+ "---\ntitle: Bob\ncategory: profile\ntags: [person]\n---\n"
+ "# Bob\n\nBob collaborates with [[Alice]] on [[Project X]].\n",
+ encoding="utf-8",
+ )
+ (vault / "topics" / "Project X").mkdir(parents=True, exist_ok=True)
+ (vault / "topics" / "Project X" / "Project X.md").write_text(
+ "---\ntitle: Project X\ncategory: concept\n---\n"
+ "# Project X\n\nA major initiative led by [[Alice]] with [[Bob]].\n",
+ encoding="utf-8",
+ )
+
+
+async def _wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> None:
+ last = -1
+ stable_for = 0
+ for _ in range(int(timeout_s / 0.25)):
+ now = len(watcher.file_store)
+ if now == last and now >= expected_min:
+ stable_for += 1
+ if stable_for >= 4:
+ return
+ else:
+ stable_for = 0
+ last = now
+ await asyncio.sleep(0.25)
+
+
+def _decode(resp) -> object:
+ """Job answers are JSON strings; decode for inspection. Pass through dicts/lists."""
+ if isinstance(resp.answer, (dict, list)):
+ return resp.answer
+ if isinstance(resp.answer, str):
+ try:
+ return json.loads(resp.answer)
+ except (json.JSONDecodeError, TypeError):
+ return resp.answer
+ return resp.answer
+
+
+async def _run_profile(name: str, config_path: Path) -> dict:
+ """Boot the profile against a fresh temp vault; exercise jobs; return summary."""
+ print(f"\n========== profile: {name} ==========")
+
+ with tempfile.TemporaryDirectory() as tmp:
+ vault = Path(tmp) / "vault"
+ vault.mkdir()
+ _seed_vault(vault)
+
+ # Override watch_path / db_path / sidecar info; force HTTP service so
+ # we never block on stdio MCP (we only call jobs directly).
+ _, cfg = parse_args(
+ "start",
+ f"config={config_path}",
+ f"components.file_watcher.default.watch_path={vault}",
+ f"components.file_store.default.db_path={vault}/.reme",
+ )
+ cfg["service"] = {"backend": "http"}
+
+ app = Application(**cfg)
+ await app.start()
+ try:
+ jobs = sorted(app.context.jobs.keys())
+ print(f" registered jobs ({len(jobs)}): {jobs}")
+
+ watcher = app.context.components["file_watcher"]["default"]
+ await _wait_for_index(watcher, expected_min=3)
+ print(f" file_store nodes after sync: {len(watcher.file_store)}")
+ assert len(watcher.file_store) >= 3, "watcher did not index seed files"
+
+ results: dict = {"jobs": jobs, "checks": []}
+ alice_path = str((vault / "topics" / "Alice" / "Alice.md").resolve())
+
+ if "memory_get" in jobs:
+ r = _decode(await app.run_job("memory_get", path=alice_path))
+ ok = isinstance(r, dict) and r.get("exists") is True
+ print(f" memory_get(Alice.md) → exists={ok}, edges={len(r.get('link', []))}")
+ results["checks"].append(("memory_get", ok))
+
+ if "memory_list" in jobs:
+ r = _decode(await app.run_job("memory_list", tags=["person"]))
+ ok = isinstance(r, dict) and r.get("count", 0) >= 2
+ print(f" memory_list(tags=[person]) → count={r.get('count') if isinstance(r, dict) else '?'}")
+ results["checks"].append(("memory_list", ok))
+
+ if "memory_search" in jobs:
+ r = _decode(await app.run_job("memory_search", query="collaborates", max_results=3, min_score=0.0))
+ hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
+ ok = len(hits) > 0
+ print(f" memory_search('collaborates') → {len(hits)} hits")
+ results["checks"].append(("memory_search", ok))
+
+ if "memory_links" in jobs:
+ r = _decode(await app.run_job("memory_links", path=alice_path))
+ ok = isinstance(r, dict) and len(r.get("links", [])) >= 2
+ print(f" memory_links(Alice.md) → {len(r.get('links', []))} resolved")
+ results["checks"].append(("memory_links", ok))
+
+ if "query" in jobs:
+ r = _decode(await app.run_job("query", query="Alice Bob", max_results=3, min_score=0.0))
+ hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
+ ok = len(hits) > 0
+ print(f" query('Alice Bob') → {len(hits)} hits")
+ results["checks"].append(("query", ok))
+
+ if "ingest" in jobs:
+ # Degraded path: no LLM key set in env → Ingestor falls back
+ # to direct create from `target_path`. That's the hermetic path.
+ target = str((vault / "topics" / "smoke" / "smoke.md").resolve())
+ r = _decode(await app.run_job(
+ "ingest",
+ content="# smoke topic\n\nrecorded by config smoke test.\n",
+ target_path=target,
+ metadata={"category": "concept", "title": "smoke"},
+ ))
+ ok = isinstance(r, dict) and (r.get("applied") or r.get("skipped"))
+ print(f" ingest(degraded create) → applied={len(r.get('applied', [])) if isinstance(r, dict) else '?'}, "
+ f"used_llm={r.get('used_llm') if isinstance(r, dict) else '?'}")
+ results["checks"].append(("ingest", bool(ok)))
+ # Confirm the file landed on disk.
+ assert Path(target).is_file(), "ingest did not produce the target file"
+
+ if "sync" in jobs:
+ # CREATE call.
+ r = _decode(await app.run_job(
+ "sync",
+ name="smoke-event",
+ description="smoke test event",
+ content="## ops\n- ran the smoke test\n",
+ topics=["[[Alice]]"],
+ tags=["smoke"],
+ materials=[
+ {"filename": "raw-prompt.md", "content": "# raw user prompt\n\nrun the smoke test\n"},
+ {"filename": "tool-output.txt", "content": "tool ran ok\nexit=0\n"},
+ ],
+ ))
+ ok = (
+ isinstance(r, dict)
+ and r.get("created") is True
+ and r.get("action") == "created"
+ and len(r.get("materials", [])) == 2
+ )
+ materials = r.get("materials", []) if isinstance(r, dict) else []
+ print(f" sync(create smoke-event w/ 2 materials) → created={r.get('created') if isinstance(r, dict) else '?'}, "
+ f"materials={len(materials)}")
+ results["checks"].append(("sync.create", bool(ok)))
+ for m in materials:
+ assert Path(m).is_file(), f"event material missing: {m}"
+ if materials:
+ event_dir = Path(materials[0]).parent
+ index_text = (event_dir / "smoke-event.md").read_text(encoding="utf-8")
+ assert "## Materials" in index_text, "index .md missing Materials section"
+ assert "raw-prompt.md" in index_text, "Materials section missing raw-prompt link"
+ await _wait_for_index(watcher, expected_min=len(watcher.file_store) + 2)
+
+ # APPEND call: same name, new content + new + colliding material.
+ r2 = _decode(await app.run_job(
+ "sync",
+ name="smoke-event",
+ content="## follow-up\n- second pass facts\n",
+ topics=["[[Bob]]"], # union with prior [[Alice]]
+ tags=["follow-up"], # union with prior [smoke]
+ materials=[
+ {"filename": "tool-output.txt", "content": "second tool run\nexit=0\n"}, # collision → auto-suffix
+ {"filename": "summary.md", "content": "# summary\nsecond pass\n"},
+ ],
+ ))
+ ok2 = (
+ isinstance(r2, dict)
+ and r2.get("created") is False
+ and r2.get("action") == "appended"
+ and len(r2.get("materials", [])) == 2
+ )
+ appended_paths = r2.get("materials", []) if isinstance(r2, dict) else []
+ print(f" sync(append smoke-event w/ collision) → action={r2.get('action') if isinstance(r2, dict) else '?'}, "
+ f"new_materials={len(appended_paths)}")
+ results["checks"].append(("sync.append", bool(ok2)))
+ # Collision should have produced tool-output-2.txt; summary.md untouched.
+ names_appended = {Path(p).name for p in appended_paths}
+ assert "tool-output-2.txt" in names_appended, f"collision auto-suffix missing: {names_appended}"
+ assert "summary.md" in names_appended, f"clean filename missing: {names_appended}"
+ # Index should now contain BOTH the original ops section and the Update section.
+ if materials:
+ index_text2 = (Path(materials[0]).parent / "smoke-event.md").read_text(encoding="utf-8")
+ assert "## ops" in index_text2, "original content lost on append"
+ assert "## Update —" in index_text2, "missing Update section after append"
+ assert "follow-up" in index_text2, "appended content not in index"
+ # Frontmatter union check
+ assert "[[Bob]]" in index_text2, "topic union failed"
+ # Materials footer should now list 4 files (2 original + summary + tool-output-2)
+ assert "tool-output-2.txt" in index_text2, "Materials footer missing collided file"
+ assert "summary.md" in index_text2, "Materials footer missing new file"
+
+ # REFUSAL: flip status to distilled, third sync should refuse.
+ index_path = str(Path(materials[0]).parent / "smoke-event.md") if materials else None
+ if index_path and "memory_property_update" in jobs:
+ await app.run_job("memory_property_update", path=index_path, key="status", value="distilled")
+ r3 = _decode(await app.run_job(
+ "sync",
+ name="smoke-event",
+ content="should refuse",
+ ))
+ ok3 = isinstance(r3, dict) and "error" in r3 and r3.get("status") == "distilled"
+ print(f" sync(refuse on distilled) → error={'error' in (r3 if isinstance(r3, dict) else {})}, "
+ f"suggested_name={r3.get('suggested_name') if isinstance(r3, dict) else '?'}")
+ results["checks"].append(("sync.refuse_distilled", bool(ok3)))
+
+ if "topic_create" in jobs:
+ r = _decode(await app.run_job(
+ "topic_create",
+ folder="Carol",
+ name="Carol",
+ category="profile",
+ description="smoke test",
+ content="# Carol\n",
+ tags=["person"],
+ ))
+ ok = isinstance(r, dict) and (r.get("created") is True or "path" in r)
+ print(f" topic_create(Carol) → created={r.get('created') if isinstance(r, dict) else '?'}")
+ results["checks"].append(("topic_create", bool(ok)))
+
+ return results
+ finally:
+ await app.close()
+
+
+async def _main() -> int:
+ summary: dict[str, dict] = {}
+ for name, path in PROFILES.items():
+ try:
+ summary[name] = await _run_profile(name, path)
+ except Exception as e:
+ print(f" ✗ profile '{name}' failed: {type(e).__name__}: {e}")
+ import traceback
+ traceback.print_exc()
+ summary[name] = {"error": str(e)}
+
+ print("\n========== summary ==========")
+ failed = 0
+ for name, result in summary.items():
+ if "error" in result:
+ print(f" {name}: ERROR — {result['error']}")
+ failed += 1
+ continue
+ checks = result.get("checks", [])
+ passed = sum(1 for _, ok in checks if ok)
+ total = len(checks)
+ marker = "✓" if passed == total else "✗"
+ print(f" {marker} {name}: {passed}/{total} checks passed; jobs={len(result['jobs'])}")
+ for label, ok in checks:
+ if not ok:
+ print(f" ✗ {label}")
+ failed += 1
+ return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+ sys.exit(asyncio.run(_main()))
diff --git a/reme2/enumeration/component_enum.py b/reme2/enumeration/component_enum.py
index 7f74a167..942c7696 100644
--- a/reme2/enumeration/component_enum.py
+++ b/reme2/enumeration/component_enum.py
@@ -24,9 +24,11 @@ class ComponentEnum(str, Enum):
EMBEDDING_MODEL = "embedding_model"
+ EDGE_EXTRACTOR = "edge_extractor"
+
FILE_PARSER = "file_parser"
- CHUNK_STORE = "chunk_store"
+ FILE_STORE = "file_store"
FILE_WATCHER = "file_watcher"
diff --git a/reme2/file_based/__init__.py b/reme2/file_based/__init__.py
deleted file mode 100644
index 81b58f52..00000000
--- a/reme2/file_based/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""File-based components and utilities."""
-
-from .file_io import FileIO
-from .file_utils import (
- async_read_file_safe,
- truncate_text_output,
-)
-from .memory_search import MemorySearch
-from .summarizer import Summarizer
-
-__all__ = [
- "FileIO",
- "async_read_file_safe",
- "truncate_text_output",
- "MemorySearch",
- "Summarizer",
-]
diff --git a/reme2/file_based/file_io.py b/reme2/file_based/file_io.py
deleted file mode 100644
index 22399015..00000000
--- a/reme2/file_based/file_io.py
+++ /dev/null
@@ -1,361 +0,0 @@
-"""File I/O operations with a configurable working directory."""
-
-import os
-from pathlib import Path
-
-import aiofiles
-from agentscope.message import TextBlock
-from agentscope.tool import ToolResponse
-
-from .file_utils import async_read_file_safe, truncate_text_output
-from ..constants import TRUNCATION_NOTICE_MARKER
-
-
-class FileIO:
- """File I/O operations with a configurable working directory."""
-
- def __init__(self, working_dir: str | Path):
- """Initialize FileIO with a working directory.
-
- Args:
- working_dir (`str`):
- The working directory for resolving relative paths.
- """
- self.working_dir = Path(working_dir)
-
- def _resolve_file_path(self, file_path: str) -> str:
- """Resolve file path: use absolute path as-is,
- resolve relative path from working_dir.
-
- Args:
- file_path: The input file path (absolute or relative).
-
- Returns:
- The resolved absolute file path as string.
- """
- path = Path(file_path).expanduser()
- if path.is_absolute():
- return str(path)
- else:
- return str(self.working_dir / file_path)
-
- async def read_file( # pylint: disable=too-many-return-statements
- self,
- file_path: str,
- start_line: int | None = None,
- end_line: int | None = None,
- ) -> ToolResponse:
- """Read a file. Relative paths resolve from WORKING_DIR.
-
- Use start_line/end_line to read a specific line range (output includes
- line numbers). Omit both to read the full file.
-
- Args:
- file_path (`str`):
- Path to the file.
- start_line (`int`, optional):
- First line to read (1-based, inclusive).
- end_line (`int`, optional):
- Last line to read (1-based, inclusive).
- """
-
- # Convert start_line/end_line to int if they are strings
- if start_line is not None:
- try:
- start_line = int(start_line)
- except (ValueError, TypeError):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: start_line must be an integer, got {start_line!r}.",
- ),
- ],
- )
-
- if end_line is not None:
- try:
- end_line = int(end_line)
- except (ValueError, TypeError):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: end_line must be an integer, got {end_line!r}.",
- ),
- ],
- )
-
- file_path = self._resolve_file_path(file_path)
-
- if not os.path.exists(file_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The file {file_path} does not exist.",
- ),
- ],
- )
-
- if not os.path.isfile(file_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The path {file_path} is not a file.",
- ),
- ],
- )
-
- try:
- content = await async_read_file_safe(file_path)
- all_lines = content.split("\n")
- total = len(all_lines)
-
- # Determine read range
- s = max(1, start_line if start_line is not None else 1)
- e = min(total, end_line if end_line is not None else total)
-
- if s > total:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: start_line {s} exceeds file length ({total} lines).",
- ),
- ],
- )
-
- if s > e:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: start_line ({s}) > end_line ({e}).",
- ),
- ],
- )
-
- # Extract selected lines
- selected_content = "\n".join(all_lines[s - 1: e])
-
- # Apply smart truncation (consistent with shell output format)
- text = truncate_text_output(
- selected_content,
- start_line=s,
- total_lines=total,
- file_path=file_path,
- )
-
- # Add continuation hint if partial read without truncation.
- # Use TRUNCATION_NOTICE_MARKER format so ToolResultCompactor can
- # re-truncate with the correct start_line when compacting old messages.
- if text == selected_content and e < total:
- content_bytes = len(text.encode("utf-8"))
- notice = (
- TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated."
- f"\nThe full content is saved to the file "
- f"and contains {total} lines in total."
- f"\nThis excerpt starts at line {s} and "
- f"covers the next {content_bytes} bytes."
- "\nIf the current content is not enough, "
- f"call `read_file` with file_path={file_path} start_line={e + 1} to read more."
- )
- text = text + notice
-
- return ToolResponse(
- content=[TextBlock(type="text", text=text)],
- )
-
- except Exception as e:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: Read file failed due to \n{e}",
- ),
- ],
- )
-
- async def write_file(
- self,
- file_path: str,
- content: str,
- ) -> ToolResponse:
- """Create or overwrite a file. Relative paths resolve from working_dir.
-
- Args:
- file_path (`str`):
- Path to the file.
- content (`str`):
- Content to write.
- """
- if not file_path:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text="Error: No `file_path` provided.",
- ),
- ],
- )
-
- file_path = self._resolve_file_path(file_path)
-
- try:
- async with aiofiles.open(file_path, "w", encoding="utf-8") as file:
- await file.write(content)
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Wrote {len(content)} bytes to {file_path}.",
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: Write file failed due to \n{e}",
- ),
- ],
- )
-
- # pylint: disable=too-many-return-statements
- async def edit_file(
- self,
- file_path: str,
- old_text: str,
- new_text: str,
- ) -> ToolResponse:
- """Find-and-replace text in a file. All occurrences of old_text are
- replaced with new_text. Relative paths resolve from working_dir.
-
- Args:
- file_path (`str`):
- Path to the file.
- old_text (`str`):
- Exact text to find.
- new_text (`str`):
- Replacement text.
- """
- if not file_path:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text="Error: No `file_path` provided.",
- ),
- ],
- )
-
- resolved_path = self._resolve_file_path(file_path)
-
- if not os.path.exists(resolved_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The file {resolved_path} does not exist.",
- ),
- ],
- )
-
- if not os.path.isfile(resolved_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The path {resolved_path} is not a file.",
- ),
- ],
- )
-
- try:
- content = await async_read_file_safe(resolved_path)
- except Exception as e:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: Read file failed due to \n{e}",
- ),
- ],
- )
-
- if old_text not in content:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The text to replace was not found in {file_path}.",
- ),
- ],
- )
-
- new_content = content.replace(old_text, new_text)
- write_response = await self.write_file(file_path=resolved_path, content=new_content)
-
- if write_response.content and len(write_response.content) > 0:
- write_text = write_response.content[0].get("text", "")
- if write_text.startswith("Error:"):
- return write_response
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Successfully replaced text in {file_path}.",
- ),
- ],
- )
-
- async def append_file(
- self,
- file_path: str,
- content: str,
- ) -> ToolResponse:
- """Append content to the end of a file. Relative paths resolve from
- working_dir.
-
- Args:
- file_path (`str`):
- Path to the file.
- content (`str`):
- Content to append.
- """
- if not file_path:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text="Error: No `file_path` provided.",
- ),
- ],
- )
-
- file_path = self._resolve_file_path(file_path)
-
- try:
- async with aiofiles.open(file_path, "a", encoding="utf-8") as file:
- await file.write(content)
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Appended {len(content)} bytes to {file_path}.",
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: Append file failed due to \n{e}",
- ),
- ],
- )
diff --git a/reme2/file_based/file_utils.py b/reme2/file_based/file_utils.py
deleted file mode 100644
index e0875679..00000000
--- a/reme2/file_based/file_utils.py
+++ /dev/null
@@ -1,201 +0,0 @@
-"""Shared utilities for file and shell tools."""
-
-import re
-
-import aiofiles
-
-from ..constants import (
- DEFAULT_MAX_BYTES,
- MAX_FILE_READ_BYTES,
- TRUNCATION_NOTICE_MARKER,
-)
-
-
-def _truncate_fresh(
- text: str,
- start_line: int,
- total_lines: int,
- max_bytes: int,
- file_path: str | None,
- encoding: str,
-) -> str:
- """Truncate fresh text (no prior truncation marker) by bytes with line integrity.
-
- Slices at the byte boundary and appends a truncation notice with a continuation
- hint so callers know which line to read next.
-
- Returns the original text unchanged when it fits within max_bytes, or when the
- last line itself exceeds max_bytes (unhandled edge case).
- """
- text_bytes = text.encode(encoding)
-
- # Under the byte limit — return as-is without any modification.
- if len(text_bytes) <= max_bytes:
- return text
-
- # Slice at the byte boundary.
- # Assuming every single line is shorter than DEFAULT_MAX_BYTES, this cut always
- # lands mid-line, guaranteeing at least one complete line before the boundary.
- # Lines that exceed DEFAULT_MAX_BYTES are not handled and may be skipped entirely.
- truncated = text_bytes[:max_bytes]
- # Decode back to str; errors="ignore" drops any split multibyte character
- # at the cut boundary without raising an exception.
- result = truncated.decode(encoding, errors="ignore")
-
- # Count '\n' characters to determine how many complete lines are included.
- # The tail after the final '\n' is a partial line that will be covered by
- # the next read starting at next_line.
- newline_count = result.count("\n")
-
- # Compute the first line number not yet fully included in this chunk.
- # max(1, ...) prevents next_line from equaling start_line when a single line
- # exceeds max_bytes (newline_count == 0), which would make the caller retry
- # the same range indefinitely.
- next_line = start_line + max(1, newline_count)
-
- if next_line <= total_lines:
- # Truncation fell before the last line — continue reading from next_line.
- read_from = next_line
- elif start_line < total_lines:
- # next_line overshot total_lines, meaning the cut landed inside the last line.
- # Re-read from the start of the last line so the caller gets it in full.
- read_from = total_lines
- else:
- # start_line == total_lines: the last line itself exceeds DEFAULT_MAX_BYTES.
- # This case is outside our handled range — return without a truncation notice.
- return result
-
- notice = (
- TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated."
- f"\nThe full content is saved to the file and contains {total_lines} lines in total."
- f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes."
- f"\nIf the current content is not enough, call `read_file` with file_path={file_path or ''} "
- f"start_line={read_from} to read more."
- )
-
- return result + notice
-
-
-def _retruncate(
- text: str,
- max_bytes: int,
- encoding: str,
-) -> str:
- """Re-truncate text that was previously truncated (contains TRUNCATION_NOTICE_MARKER).
-
- Extracts the original content before the marker, applies the new byte limit, and
- updates the embedded notice (byte count and continuation line number) via regex.
-
- Returns the original text unchanged when:
- - the content already fits within max_bytes (with a small slack);
- - required metadata fields cannot be parsed from the existing notice.
- """
- parts = text.split(TRUNCATION_NOTICE_MARKER, 1)
- original_content = parts[0]
- old_notice = parts[1]
-
- text_bytes = original_content.encode(encoding)
-
- # Allow a small slack to avoid unnecessary re-truncation when content is just
- # barely over the limit (e.g. due to minor encoding differences).
- if len(text_bytes) <= max_bytes + 100:
- return text
-
- # Parse start_line from notice; return text unchanged if not found
- start_match = re.search(r"starts at line (\d+)", old_notice)
- if not start_match:
- return text
- start_line_parsed = int(start_match.group(1))
-
- # Re-slice to the new byte limit.
- # Because every line is assumed to be shorter than DEFAULT_MAX_BYTES, the cut
- # always falls somewhere mid-line, so at least one complete line is preserved.
- truncated_bytes = text_bytes[:max_bytes]
- # errors="ignore" silently drops any incomplete multibyte character at the cut boundary.
- result = truncated_bytes.decode(encoding, errors="ignore")
- # Each '\n' in result corresponds to one fully-included line;
- # anything after the last '\n' is a partial line that was cut off.
- newline_count = result.count("\n")
-
- # The next read should start at the line immediately after all complete lines.
- # max(1, ...) guards against the theoretical zero-newline case
- # (impossible when every line is shorter than DEFAULT_MAX_BYTES).
- next_line = start_line_parsed + max(1, newline_count)
-
- if not re.search(r"covers the next \d+ bytes", old_notice):
- return text
- # _truncate_fresh always includes a continuation hint, so both fields are always present.
- new_notice = re.sub(r"covers the next \d+ bytes", f"covers the next {max_bytes} bytes", old_notice)
- new_notice = re.sub(r"start_line=\d+ to read more", f"start_line={next_line} to read more", new_notice)
-
- return result + TRUNCATION_NOTICE_MARKER + new_notice
-
-
-def truncate_text_output(
- text: str,
- start_line: int = 1,
- total_lines: int = 0,
- max_bytes: int = DEFAULT_MAX_BYTES,
- file_path: str | None = None,
- encoding: str = "utf-8",
-) -> str:
- """Truncate file output by bytes with line integrity.
-
- If text is under byte limit, return as-is.
- If over limit, truncate at the last complete line that fits,
- allowing the next read to start from a fresh line.
-
- Dispatches to :func:`_truncate_fresh` for text seen for the first time, or to
- :func:`_retruncate` when the text already contains a TRUNCATION_NOTICE_MARKER
- from a previous pass.
-
- Args:
- text: The output text to truncate.
- start_line: The starting line number (1-based). Ignored when text already
- contains a truncation notice (values are parsed from the notice instead).
- total_lines: Total lines in the original file. Ignored when text already
- contains a truncation notice (values are parsed from the notice instead).
- max_bytes: Maximum size in bytes.
- file_path: Optional file path to include in the truncation notice.
- encoding: Character encoding used for byte-length calculation and decoding.
-
- Returns:
- Truncated text with notice if truncated.
- """
- if not text:
- return text
- if max_bytes <= 0:
- return text
-
- try:
- if TRUNCATION_NOTICE_MARKER in text:
- return _retruncate(text, max_bytes=max_bytes, encoding=encoding)
- else:
- return _truncate_fresh(
- text,
- start_line=start_line,
- total_lines=total_lines,
- max_bytes=max_bytes,
- file_path=file_path,
- encoding=encoding,
- )
- except Exception:
- return text
-
-
-async def async_read_file_safe(file_path: str, max_bytes: int = MAX_FILE_READ_BYTES) -> str:
- """Async version of read_file_safe with Unicode error handling and memory protection.
-
- Args:
- file_path: Path to the file.
- max_bytes: Maximum bytes to read into memory (default 1GB).
-
- Returns:
- File content as string (up to max_bytes).
- """
- try:
- async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
- return await f.read(max_bytes)
- except UnicodeDecodeError:
- async with aiofiles.open(file_path, "r", encoding="utf-8", errors="ignore") as f:
- return await f.read(max_bytes)
diff --git a/reme2/file_based/memory_search.py b/reme2/file_based/memory_search.py
deleted file mode 100644
index 4abb7507..00000000
--- a/reme2/file_based/memory_search.py
+++ /dev/null
@@ -1,56 +0,0 @@
-"""Memory search step for semantic search in memory files."""
-
-import json
-
-from ..component import R
-from ..component.base_step import BaseStep
-
-
-@R.register("memory_search")
-class MemorySearch(BaseStep):
- """Semantically search MEMORY.md and memory files."""
-
- def __init__(self, vector_weight: float = 0.7, candidate_multiplier: float = 3.0, **kwargs):
- """Initialize memory search step.
-
- Args:
- vector_weight: Weight for vector search vs keyword search.
- candidate_multiplier: Multiplier for candidate count before filtering.
- **kwargs: Additional arguments passed to BaseStep.
- """
- super().__init__(**kwargs)
- self.vector_weight = vector_weight
- self.candidate_multiplier = candidate_multiplier
-
- async def execute(self):
- """Execute the memory search operation."""
- assert self.context is not None, "Context is not set"
- query: str = self.context.get("query", "").strip()
- min_score: float = self.context.get("min_score", 0.1)
- max_results: int = self.context.get("max_results", 5)
-
- assert query, "Query cannot be empty"
- assert (
- isinstance(min_score, float | int) and 0.0 <= min_score <= 1.0
- ), f"min_score must be between 0 and 1, got {min_score}"
- assert (
- isinstance(max_results, int) and max_results > 0
- ), f"max_results must be a positive integer, got {max_results}"
-
- chunk_filter = self.file_graph.filter(
- paths=self.context.get("paths") or None,
- tags=self.context.get("tags") or None,
- exclude_paths=self.context.get("exclude_paths") or None,
- )
-
- results = await self.chunk_store.hybrid_search(
- query=query,
- limit=max_results,
- vector_weight=self.vector_weight,
- candidate_multiplier=self.candidate_multiplier,
- chunk_filter=chunk_filter,
- )
-
- results = [r for r in results if r.score >= min_score]
-
- return json.dumps([result.model_dump(exclude_none=True) for result in results], indent=2, ensure_ascii=False)
diff --git a/reme2/mcp/README.md b/reme2/mcp/README.md
new file mode 100644
index 00000000..7ae4b705
--- /dev/null
+++ b/reme2/mcp/README.md
@@ -0,0 +1,73 @@
+# reme2.mcp
+
+Agent-facing MCP interface layer. Exposes the markdown vault under
+`reme2/` (file_store + watcher + memory services) as MCP tools for
+`claude-code` and other MCP clients.
+
+## Layout
+
+```
+reme2/mcp/
+├── __init__.py
+├── server.py MCP server bootstrap; defaults to ../config/full.yaml
+└── steps/ @R.register MCP step shells
+ ├── memory_io.py memory_create/update/get/list/links/...
+ ├── memory_retriever.py memory_search + memory_graph_search
+ ├── sync.py hot-path event-folder upsert
+ └── topic_create.py typed topic creation w/ schema gates
+```
+
+The MCP layer's job is to **project** existing primitives as MCP tools
+— it owns no business logic. Everything else lives outside the
+transport boundary so memory services don't form an import cycle:
+
+| Concern | Location |
+|---|---|
+| Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` |
+| Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer |
+| Pure write helpers + agent toolkit | `reme2/memory/memory_io.py` |
+| Vault domain models (Event, Topic) | `reme2/schema/vault/` |
+| Path templates + name disambiguation | `reme2/utils/vault_paths.py` |
+| Step response serialization | `reme2/component/runtime_response.py` |
+
+Dependency direction is strict:
+```
+reme2.mcp → reme2.memory → reme2.component / reme2.schema / reme2.utils
+```
+
+## Run
+
+```bash
+# stdio MCP server, full tool surface
+python -m reme2.mcp.server
+
+# override config or any field
+python -m reme2.mcp.server config=reme2/config/curated.yaml
+python -m reme2.mcp.server components.file_watcher.default.watch_path=/abs/vault
+```
+
+Config profiles (in `reme2/config/`):
+- `full.yaml` — every memory_* primitive + sync + topic_create + ingest
+- `curated.yaml` — opinionated 3-tool surface (`query`, `sync`, `ingest`)
+
+## Tools exposed (full profile)
+
+| Tool | Path | Purpose |
+|---|---|---|
+| `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
+| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation. |
+| `topic_create` | steps/topic_create.py | Typed topic creation with schema gates. |
+| `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. |
+| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | steps/memory_io.py | Read primitives. |
+| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | steps/memory_io.py | Raw write primitives (prefer `ingest` / `sync`). |
+| `memory_count_tokens` | steps/memory_io.py | Token estimation. |
+
+## Smoke test
+
+```bash
+python reme2/config/smoke_test.py
+```
+
+Boots both `full` and `curated` profiles in a temp vault, exercises a
+representative subset of jobs end-to-end (in-process — no MCP
+transport).
diff --git a/reme2/mcp/__init__.py b/reme2/mcp/__init__.py
new file mode 100644
index 00000000..2a107aae
--- /dev/null
+++ b/reme2/mcp/__init__.py
@@ -0,0 +1,20 @@
+"""reme2.mcp — MCP interface layer.
+
+The Agent-facing surface: server entrypoint + step shells that wrap
+the three services in `reme2.memory` (Retriever, Ingestor, Maintainer)
+plus hot-write primitives (sync, topic_create, memory_*) that bypass
+services and land directly on the Memory File System.
+
+This package depends on `reme2.memory`, `reme2.schema.vault`,
+`reme2.utils`, `reme2.component` — never the reverse. Domain types
+(Topic / Event), pure helpers (path builders, naming), and the write
+primitives all live outside `mcp/` so memory-layer services can use
+them without forming an import cycle through the transport layer.
+
+Sub-packages:
+ steps/ - all @R.register MCP step shells (memory_io, memory_retriever,
+ sync, topic_create).
+ server.py - MCP server bootstrap (defaults to ../config/full.yaml).
+"""
+
+__version__ = "0.1.0"
diff --git a/reme2/mcp/server.py b/reme2/mcp/server.py
new file mode 100644
index 00000000..2a4f8c74
--- /dev/null
+++ b/reme2/mcp/server.py
@@ -0,0 +1,74 @@
+"""vault MCP server entry point.
+
+Usage:
+ python -m reme2.mcp.server [config=path/to/yaml] [service.transport=stdio]
+ python reme2/mcp/server.py [config=path/to/yaml] [service.transport=stdio]
+
+By default loads `reme2/config/full.yaml` and starts the
+ReMe2 application with the MCP service registered.
+
+Env vars:
+ VAULT_PATH — override vault watch path (overrides config + cli args).
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Make this work whether invoked as `python -m reme2.mcp.server` (repo
+# root already on sys.path) or `python reme2/mcp/server.py` (only this
+# file's dir is on sys.path — without this, `import reme2.mcp.steps`
+# would fail with ModuleNotFoundError).
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+from dotenv import load_dotenv # noqa: E402
+
+# Load .env from repo root before anything else touches env vars
+# (embedding/llm clients read them at construction time).
+load_dotenv(_REPO_ROOT / ".env")
+
+# Eagerly import side-effect modules so all @R.register() decorators run
+# before Application introspects the registry.
+import reme2 # noqa: E402,F401
+import reme2.mcp.steps # noqa: E402,F401
+import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
+import reme2.memory.ingestor # noqa: E402,F401
+import reme2.memory.summarizer # noqa: E402,F401
+import reme2.component.service.mcp_service # noqa: E402,F401
+
+from reme2.application import Application # noqa: E402
+from reme2.config import parse_args # noqa: E402
+
+
+_DEFAULT_CONFIG = str(_REPO_ROOT / "reme2" / "config" / "full.yaml")
+
+
+def main() -> None:
+ argv = list(sys.argv[1:])
+ if not argv or argv[0].startswith("config=") or "=" in argv[0]:
+ argv.insert(0, "start")
+ if not any(a.startswith("config=") for a in argv[1:]):
+ argv.insert(1, f"config={_DEFAULT_CONFIG}")
+
+ vault_path = os.environ.get("VAULT_PATH")
+ if vault_path:
+ argv.append(f"components.file_watcher.default.watch_path={vault_path}")
+ argv.append(f"components.file_store.default.db_path={vault_path}/.reme")
+ argv.append(f"service.sidecar_info_path={vault_path}/.reme/sidecar.json")
+
+ sidecar_port = os.environ.get("VAULT_HTTP_PORT")
+ if sidecar_port:
+ argv.append(f"service.sidecar_http_port={sidecar_port}")
+
+ action, config = parse_args(*argv)
+ if action != "start":
+ raise SystemExit("reme2.mcp.server only supports the 'start' action")
+
+ app = Application(**config)
+ app.run_app()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/reme2/mcp/steps/__init__.py b/reme2/mcp/steps/__init__.py
new file mode 100644
index 00000000..1b7a0e70
--- /dev/null
+++ b/reme2/mcp/steps/__init__.py
@@ -0,0 +1,31 @@
+"""MCP step shells — the @R.register classes the model invokes as MCP tools.
+
+Two groups:
+
+ Hot-write primitives that bypass services and land on MFS directly:
+ sync — idempotent event-folder upsert (event log)
+ topic_create — typed topic creation (hard schema gates)
+ memory_io — memory_create / update / property_update / rename /
+ delete / archive / get / list / links / backlinks /
+ resolve_wikilink / count_tokens
+
+ Service delegate:
+ memory_retriever — memory_search + memory_graph_search; thin
+ wrappers that delegate to the configured
+ Retriever component (`reme2.memory.retriever`).
+
+The Ingestor's MCP face (`ingest`) is registered from
+`reme2.memory.ingestor`, since it's a service step rather than an MFS
+primitive. Importing this package triggers all the step registrations
+hosted here.
+"""
+
+from . import memory_io # noqa: F401 -- triggers @R.register for memory_*
+from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search
+from .sync import Sync
+from .topic_create import TopicCreate
+
+__all__ = [
+ "Sync",
+ "TopicCreate",
+]
diff --git a/reme2/mcp/steps/memory_io.py b/reme2/mcp/steps/memory_io.py
new file mode 100644
index 00000000..037930fb
--- /dev/null
+++ b/reme2/mcp/steps/memory_io.py
@@ -0,0 +1,208 @@
+"""MCP step shells over the Memory File System engine API.
+
+Per `structure.md`, .md files are the SSOT and the engine surface lives
+in `reme2.memory.memory_io` (CRUD writes + MFS reads + Projections).
+This module only hosts the `@R.register("memory_*")` Step shells —
+each one translates RuntimeContext ↔ JSON payload and delegates to the
+matching engine API function.
+
+Search ops live in `memory_retriever.py` (Retriever composes V/K/graph
+projections with policy).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ...component import R
+from ...component.base_step import BaseStep
+from ...component.runtime_response import _set_answer
+from ...enumeration import ComponentEnum
+from ...memory import memory_io
+
+
+@R.register("memory_get")
+class MemoryGet(BaseStep):
+ """Read a single memory file (frontmatter + body, optional chunks)."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ include_chunks: bool = bool(self.context.get("include_chunks", False))
+ assert path, "path is required"
+ result = await memory_io.read_file(self.file_store, path, include_chunks=include_chunks)
+ _set_answer(self.context, result)
+
+
+@R.register("memory_list")
+class MemoryList(BaseStep):
+ """List indexed files filtered by frontmatter fields, tags, or path prefix."""
+
+ async def execute(self):
+ assert self.context is not None
+ result = memory_io.list_files(
+ self.file_store,
+ path_prefix=self.context.get("path_prefix"),
+ tags=self.context.get("tags") or [],
+ metadata=self.context.get("metadata") or {},
+ limit=int(self.context.get("limit", 100)),
+ )
+ _set_answer(self.context, result)
+
+
+@R.register("memory_backlinks")
+class MemoryBacklinks(BaseStep):
+ """Files linking to a given path. Each entry carries the typed-edge predicate."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ assert path, "path is required"
+ _set_answer(self.context, memory_io.backlinks_of(self.file_store, path))
+
+
+@R.register("memory_links")
+class MemoryLinks(BaseStep):
+ """Files a given path links to (resolved). Each entry carries the typed-edge predicate."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ assert path, "path is required"
+ _set_answer(self.context, memory_io.links_of(self.file_store, path))
+
+
+@R.register("memory_resolve_wikilink")
+class MemoryResolveWikilink(BaseStep):
+ """Resolve a `[[target]]` wikilink with full ambiguity context."""
+
+ async def execute(self):
+ assert self.context is not None
+ wikilink: str = self.context.get("wikilink", "") or ""
+ assert wikilink, "wikilink is required"
+ payload = memory_io.wikilink_lookup(self.file_store, wikilink)
+ self.context.response.success = bool(payload.get("exists"))
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_create")
+class MemoryCreate(BaseStep):
+ """Create a markdown file. Wikilink-uniqueness gate runs unless force=True."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ metadata: dict = dict(self.context.get("metadata", {}) or {})
+ content: str = self.context.get("content", "") or ""
+ overwrite: bool = bool(self.context.get("overwrite", False))
+ force: bool = bool(self.context.get("force", False))
+
+ assert path, "path is required"
+ target = Path(path)
+
+ ok, payload = memory_io.write_create(
+ self.file_store, target,
+ metadata=metadata, content=content,
+ overwrite=overwrite, force=force,
+ )
+ self.context.response.success = ok
+ if ok:
+ payload = {**payload, "path": str(target.resolve())}
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_delete")
+class MemoryDelete(BaseStep):
+ """Delete a file. Watcher removes from store + graph."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ assert path, "path is required"
+ ok, payload = memory_io.write_delete(path)
+ self.context.response.success = ok
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_rename")
+class MemoryRename(BaseStep):
+ """Rename a file and rewrite incoming wikilinks across the vault."""
+
+ async def execute(self):
+ assert self.context is not None
+ old_path: str = self.context.get("old_path", "")
+ new_path: str = self.context.get("new_path", "")
+ assert old_path and new_path, "old_path and new_path are required"
+
+ watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[index,union-attr]
+ vault_root = Path(watcher.watch_path).resolve() # type: ignore[union-attr]
+
+ ok, payload = memory_io.write_rename(self.file_store, vault_root, old_path, new_path)
+ self.context.response.success = ok
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_property_update")
+class MemoryPropertyUpdate(BaseStep):
+ """Update a single YAML frontmatter key. value=null deletes the key."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ key: str = self.context.get("key", "")
+ value = self.context.get("value")
+ assert path and key, "path and key are required"
+ ok, payload = memory_io.write_property_update(path, key, value)
+ self.context.response.success = ok
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_update")
+class MemoryUpdate(BaseStep):
+ """Edit-style content update: replace `old_string` with `new_string`."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "")
+ old_string: str = self.context.get("old_string", "")
+ new_string: str = self.context.get("new_string", "")
+ replace_all: bool = bool(self.context.get("replace_all", False))
+ assert path, "path is required"
+ ok, payload = memory_io.write_update(path, old_string, new_string, replace_all=replace_all)
+ self.context.response.success = ok
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_archive")
+class MemoryArchive(BaseStep):
+ """Archive a file: flip `status: archived` then move to `/Archive/`."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "") or ""
+ archive_dir_name: str = self.context.get("archive_dir", "Archive") or "Archive"
+ assert path, "path is required"
+
+ watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
+ vault_root = Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd()
+
+ ok, payload = memory_io.write_archive(vault_root, path, archive_dir_name)
+ self.context.response.success = ok
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_count_tokens")
+class MemoryCountTokens(BaseStep):
+ """Estimate tokens for a file body or raw text. One of `path`/`text` required."""
+
+ async def execute(self):
+ assert self.context is not None
+ path: str = self.context.get("path", "") or ""
+ text: str = self.context.get("text", "") or ""
+ result = await memory_io.count_tokens(
+ self.as_token_counter,
+ path=path or None,
+ text=text or None,
+ )
+ self.context.response.success = "error" not in result
+ _set_answer(self.context, result)
diff --git a/reme2/mcp/steps/memory_retriever.py b/reme2/mcp/steps/memory_retriever.py
new file mode 100644
index 00000000..d4046c6d
--- /dev/null
+++ b/reme2/mcp/steps/memory_retriever.py
@@ -0,0 +1,199 @@
+"""Memory retriever steps — thin MCP-facing wrappers over `BaseRetriever`.
+
+Per the architecture blueprint, retrieval policy (V+K hybrid + graph
+BFS fusion + ranking + intent routing) is the **Retriever service**
+(`reme2.memory.retriever`), registered as `ComponentEnum.RETRIEVER`.
+These steps are the MCP projection: they translate `RuntimeContext`
+(paths/tags/exclude_paths filter, per-call knob overrides) into the
+retriever's call surface, then serialize results for the API response
+(joining file metadata onto each chunk so callers don't have to fire a
+`memory_get` per hit).
+
+Direct primary-key file ops (read/list/links/backlinks) live in
+`memory_io` — those aren't retrieval.
+"""
+
+from __future__ import annotations
+
+from ...component import R
+from ...component.base_step import BaseStep
+from ...component.runtime_response import _set_answer
+from ...enumeration import ComponentEnum
+from ...memory import memory_io
+from ...memory.retriever import BaseRetriever, HybridRetriever
+
+
+# Per-shell singleton cache: instantiating the retriever is cheap (it
+# just stashes constructor knobs), but doing it once per call would
+# still allocate every request. Keyed by step instance so each MCP
+# job's overrides stay isolated.
+_RETRIEVER_CACHE: dict[int, BaseRetriever] = {}
+
+
+def _resolve_retriever(step: BaseStep) -> BaseRetriever:
+ """Get (or build) the retriever instance for this MCP step.
+
+ The retriever is a registered Step (`@R.register("hybrid")`), but
+ it isn't pre-instantiated as a singleton component — there's no
+ RETRIEVER enum slot. Instead, each MCP shell builds its own
+ HybridRetriever the first time it's called, sharing the calling
+ step's `app_context` (so the lookup of `file_store`/`as_llm` works)
+ and forwarding any retriever knobs (`vector_weight`, `graph_*`, …)
+ from the step's kwargs as constructor defaults.
+
+ Callers can also pass a pre-built `BaseRetriever` instance via
+ `kwargs["retriever"]` — useful for tests / Python callers that want
+ to inject a stub.
+ """
+ injected = step.kwargs.get("retriever")
+ if isinstance(injected, BaseRetriever):
+ return injected
+
+ cached = _RETRIEVER_CACHE.get(id(step))
+ if cached is not None:
+ return cached
+
+ backend = injected if isinstance(injected, str) else "hybrid"
+ cls = R.get(ComponentEnum.STEP, backend)
+ if cls is None or not (isinstance(cls, type) and issubclass(cls, BaseRetriever)):
+ # Fall back to the canonical implementation; lets configs that
+ # don't override `retriever` work out of the box.
+ cls = HybridRetriever
+
+ knob_keys = (
+ "vector_weight", "graph_weight", "graph_depth", "graph_decay",
+ "graph_direction", "graph_mode", "graph_per_path_cap",
+ "candidate_multiplier", "anchor_expand", "file_store",
+ )
+ init_kwargs = {k: step.kwargs[k] for k in knob_keys if k in step.kwargs}
+ init_kwargs["app_context"] = step.app_context
+
+ instance = cls(**init_kwargs)
+ _RETRIEVER_CACHE[id(step)] = instance
+ return instance
+
+
+def _serialize_chunk(chunk, file_store, extras: dict | None = None) -> dict:
+ """Serialize a FileChunk for search-result payloads.
+
+ Joins the owning file's metadata (frontmatter + st_mtime) so callers
+ don't have to fire a `memory_get` per result just to read `category`,
+ `status`, dates, etc. The join is a single in-memory dict lookup —
+ cost is negligible vs. the round-trip we save.
+
+ `extras` lets the caller attach step-specific fields (e.g. `graph_hop`).
+ """
+ item = chunk.model_dump(exclude_none=True, exclude={"embedding"})
+ meta = file_store.get_file_meta(chunk.path)
+ if meta is not None:
+ item["file_metadata"] = meta.metadata
+ item["file_st_mtime"] = meta.st_mtime
+ else:
+ item["file_metadata"] = None
+ item["file_st_mtime"] = None
+ if extras:
+ item.update(extras)
+ return item
+
+
+@R.register("memory_search")
+class MemorySearch(BaseStep):
+ """Pure-relevance retrieval (V + K hybrid). Delegates to the Retriever service."""
+
+ async def execute(self):
+ assert self.context is not None, "Context is not set"
+ query: str = self.context.get("query", "").strip()
+ min_score: float = self.context.get("min_score", 0.1)
+ max_results: int = self.context.get("max_results", 5)
+
+ assert query, "Query cannot be empty"
+ assert (
+ isinstance(min_score, float | int) and 0.0 <= min_score <= 1.0
+ ), f"min_score must be between 0 and 1, got {min_score}"
+ assert (
+ isinstance(max_results, int) and max_results > 0
+ ), f"max_results must be a positive integer, got {max_results}"
+
+ chunk_filter = memory_io.make_chunk_filter(
+ self.file_store,
+ paths=self.context.get("paths") or None,
+ tags=self.context.get("tags") or None,
+ exclude_paths=self.context.get("exclude_paths") or None,
+ )
+
+ retriever = _resolve_retriever(self)
+ results = await retriever.search(
+ query=query,
+ max_results=max_results,
+ min_score=min_score,
+ chunk_filter=chunk_filter,
+ )
+
+ payload = [_serialize_chunk(r, self.file_store) for r in results]
+ _set_answer(self.context, payload)
+
+
+@R.register("memory_graph_search")
+class MemoryGraphSearch(BaseStep):
+ """V + K + graph BFS fusion. Delegates to the Retriever service.
+
+ Per-call overrides for fusion knobs (vector_weight, graph_weight,
+ graph_depth, graph_decay, graph_direction, graph_mode,
+ graph_per_path_cap, anchor_expand) are forwarded if present in the
+ RuntimeContext; otherwise the retriever falls back to its own
+ constructor defaults. These knobs are intentionally NOT exposed via
+ MCP — they're internal-Python-caller tuning.
+ """
+
+ _OVERRIDE_KEYS = (
+ "vector_weight",
+ "graph_weight",
+ "graph_depth",
+ "graph_decay",
+ "graph_direction",
+ "graph_mode",
+ "graph_per_path_cap",
+ "anchor_expand",
+ )
+
+ async def execute(self):
+ assert self.context is not None
+ ctx = self.context
+
+ query: str = ctx.get("query", "").strip()
+ max_results: int = int(ctx.get("max_results", 5))
+ min_score: float = float(ctx.get("min_score", 0.0))
+ explicit_seeds: list[str] = list(ctx.get("seeds") or [])
+
+ assert query or explicit_seeds, "query or seeds must be provided"
+ assert max_results > 0
+
+ chunk_filter = memory_io.make_chunk_filter(
+ self.file_store,
+ paths=ctx.get("paths") or None,
+ tags=ctx.get("tags") or None,
+ exclude_paths=ctx.get("exclude_paths") or None,
+ )
+
+ # Forward per-call overrides only if the caller actually set them;
+ # the retriever falls back to its own defaults for missing keys.
+ overrides = {k: ctx.get(k) for k in self._OVERRIDE_KEYS if ctx.get(k) is not None}
+
+ retriever = _resolve_retriever(self)
+ results, hops = await retriever.graph_search(
+ query=query,
+ seeds=explicit_seeds,
+ max_results=max_results,
+ min_score=min_score,
+ chunk_filter=chunk_filter,
+ **overrides,
+ )
+
+ payload = []
+ for c in results:
+ extras = {}
+ hop = hops.get(c.path)
+ if hop is not None:
+ extras["graph_hop"] = hop
+ payload.append(_serialize_chunk(c, self.file_store, extras))
+ _set_answer(ctx, payload)
diff --git a/reme2/mcp/steps/sync.py b/reme2/mcp/steps/sync.py
new file mode 100644
index 00000000..09eb81d7
--- /dev/null
+++ b/reme2/mcp/steps/sync.py
@@ -0,0 +1,418 @@
+"""sync — continuously sync key materials into an event folder.
+
+Layout (one folder per logical thread):
+
+ events/{date}/{name}/
+ ├── {name}.md # index: frontmatter + narrative + Materials footer
+ ├── {material_filename} # raw artifact 1
+ ├── ...
+ └── {material_filename} # raw artifact N
+
+Hot-path write entry. The agent picks a stable `name` per logical
+thread and calls `sync` repeatedly through the task — each call
+extends the same event folder rather than creating a new one. This
+turns discrete writes into a coherent stream and lets PreCompact /
+SessionEnd hooks treat sync as the last-chance flush before context
+loss.
+
+Behavior:
+ * If `events/{date}/{name}/` does NOT exist → create new folder,
+ write index `{name}.md` (status=active), write materials.
+ * If it exists with `status: active` → APPEND:
+ - new content → `## Update — {iso}` section appended to the body
+ - new materials → siblings (auto-suffix on filename collision)
+ - Materials footer regenerated as the trailing section, listing
+ every artifact actually present in the folder
+ - frontmatter `topics` + `tags` unioned, `updated` set to today
+ * If it exists with `status: distilled` / `archived` → REFUSE,
+ return suggested_name so the agent can start a new thread.
+
+Zero LLM cost.
+"""
+
+import json
+import os
+import re
+from datetime import date as date_type, datetime, timezone
+from pathlib import Path
+
+import frontmatter
+from pydantic import ValidationError
+
+from reme2.component import R
+from reme2.component.base_step import BaseStep
+from reme2.memory.memory_io import write_create
+from reme2.schema.vault import Event
+from reme2.utils.vault_paths import event_path, next_suffixed_stem
+
+
+_SAFE_FILENAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
+_MATERIALS_HEADER = "## Materials"
+
+
+@R.register("sync")
+class Sync(BaseStep):
+ """Upsert into an event folder under events/{date}/{name}/.
+
+ Inputs (RuntimeContext):
+ name (str, required): kebab-case event identifier; becomes both
+ the parent dir and the index filename stem.
+ Reuse the same name across calls in one
+ thread to keep extending the same folder.
+ description (str): one-line summary for index frontmatter
+ (only set on initial create).
+ content (str): markdown body. On create it's the initial
+ body; on append it's added under a new
+ `## Update — {iso}` section.
+ topics (list[str]): related topic wikilinks; merged (union)
+ into frontmatter on append.
+ tags (list[str]): free-form tags; merged (union) on append.
+ materials (list[dict]): [{filename, content}, ...] — raw artifacts
+ written as siblings of the index. Filename
+ collisions auto-suffix (foo.txt → foo-2.txt).
+ Index body's Materials footer regenerated
+ each call from actual folder contents.
+ on_date (str | None): ISO date for the events/{date}/ bucket.
+ Defaults to today.
+ origin_session_id (str): optional source session identifier (only
+ set on initial create).
+
+ Output (context.response.answer):
+ JSON {path, materials: [paths of NEW materials this call],
+ created: bool, action: "created"|"appended"} on success;
+ {error, ...} on failure (including refusal when existing event
+ has status != "active").
+ """
+
+ def __init__(self, vault_root: str = "", events_dir: str = "events", **kwargs):
+ super().__init__(**kwargs)
+ self.vault_root = vault_root
+ self.events_dir = events_dir
+
+ def _root(self) -> Path:
+ if self.vault_root:
+ return Path(self.vault_root)
+ watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[union-attr]
+ return Path(watcher.watch_path)
+
+ @staticmethod
+ def _validate_materials(materials: list, index_filename: str) -> tuple[list[dict], str | None]:
+ """Sanity-check the materials list. Returns (cleaned, error_message)."""
+ cleaned: list[dict] = []
+ seen_in_call: set[str] = set()
+ for i, m in enumerate(materials):
+ if not isinstance(m, dict):
+ return [], f"materials[{i}] must be an object {{filename, content}}"
+ fname = m.get("filename")
+ if not isinstance(fname, str) or not fname:
+ return [], f"materials[{i}].filename is required"
+ if not _SAFE_FILENAME_RE.match(fname):
+ return [], (
+ f"materials[{i}].filename {fname!r} is unsafe — only "
+ f"letters / digits / dot / underscore / dash allowed"
+ )
+ if fname == index_filename:
+ return [], f"materials[{i}].filename {fname!r} collides with the index file"
+ if fname in seen_in_call:
+ return [], f"materials[{i}].filename {fname!r} duplicated within the same call"
+ seen_in_call.add(fname)
+ content = m.get("content", "")
+ if not isinstance(content, str):
+ return [], f"materials[{i}].content must be a string"
+ cleaned.append({"filename": fname, "content": content})
+ return cleaned, None
+
+ @staticmethod
+ def _strip_materials_footer(body: str) -> str:
+ """Drop our trailing `## Materials` footer if present; return narrative."""
+ if not body:
+ return ""
+ # Match the footer at end-of-doc: `## Materials\n\n- [...](./...)\n` repeated.
+ # Cheaper rule: find the LAST `## Materials` heading at line start; strip
+ # from there to EOF. Whatever the user wrote above stays intact.
+ m = re.search(r"(?:\A|\n)##\s+Materials[ \t]*\n", body)
+ if m is None:
+ return body.rstrip()
+ # Find the LAST such heading by scanning all matches.
+ last = None
+ for hit in re.finditer(r"(?:\A|\n)##\s+Materials[ \t]*\n", body):
+ last = hit
+ assert last is not None
+ cut = last.start()
+ # If the heading was at offset 0 (no leading \n), keep nothing before;
+ # otherwise keep up to (but not including) the leading \n.
+ return body[:cut].rstrip()
+
+ @staticmethod
+ def _emit_body(narrative: str, material_filenames: list[str]) -> str:
+ """Assemble body = narrative (possibly empty) + Materials footer."""
+ narrative = (narrative or "").rstrip()
+ if not material_filenames:
+ return f"{narrative}\n" if narrative else ""
+ listing = "\n".join(f"- [{f}](./{f})" for f in material_filenames)
+ if narrative:
+ return f"{narrative}\n\n{_MATERIALS_HEADER}\n\n{listing}\n"
+ return f"{_MATERIALS_HEADER}\n\n{listing}\n"
+
+ @staticmethod
+ def _resolve_filename(existing: set[str], requested: str) -> str:
+ """Auto-suffix `foo.txt` → `foo-2.txt` (then -3, -4, …) on collision."""
+ if requested not in existing:
+ return requested
+ if "." in requested:
+ stem, _, ext = requested.rpartition(".")
+ n = 2
+ while f"{stem}-{n}.{ext}" in existing:
+ n += 1
+ return f"{stem}-{n}.{ext}"
+ n = 2
+ while f"{requested}-{n}" in existing:
+ n += 1
+ return f"{requested}-{n}"
+
+ @staticmethod
+ def _list_existing_materials(folder: Path, index_filename: str) -> list[str]:
+ """Filenames in `folder` excluding the index, sorted for stability."""
+ if not folder.is_dir():
+ return []
+ return sorted(
+ entry.name for entry in folder.iterdir()
+ if entry.is_file() and entry.name != index_filename
+ )
+
+ @staticmethod
+ def _union(prior: list, incoming: list) -> list:
+ """Order-preserving union: keep prior order, append new items in input order."""
+ out = list(prior)
+ seen = set(prior)
+ for item in incoming:
+ if item not in seen:
+ out.append(item)
+ seen.add(item)
+ return out
+
+ def _set_error(self, payload: dict) -> None:
+ assert self.context is not None
+ self.context.response.success = False
+ self.context.response.answer = json.dumps(payload, ensure_ascii=False)
+
+ def _set_ok(self, payload: dict) -> None:
+ assert self.context is not None
+ self.context.response.success = True
+ self.context.response.answer = json.dumps(payload, ensure_ascii=False)
+
+ async def execute(self):
+ assert self.context is not None
+ name: str = self.context.get("name", "") or ""
+ description: str = self.context.get("description", "") or ""
+ content: str = self.context.get("content", "") or ""
+ topics: list[str] = list(self.context.get("topics") or [])
+ tags: list[str] = list(self.context.get("tags") or [])
+ materials_in = list(self.context.get("materials") or [])
+ on_date = self.context.get("on_date")
+ origin_session_id = self.context.get("origin_session_id")
+
+ assert name, "name is required"
+
+ target = event_path(self._root(), name, on_date, self.events_dir)
+ materials, mat_err = self._validate_materials(materials_in, target.name)
+ if mat_err is not None:
+ self._set_error({"error": mat_err})
+ return
+
+ if target.exists():
+ await self._append(target, content, materials, topics, tags)
+ else:
+ await self._create(
+ target, name, description, content, materials, topics, tags,
+ on_date, origin_session_id,
+ )
+
+ async def _create(
+ self, target: Path, name: str, description: str, content: str,
+ materials: list[dict], topics: list[str], tags: list[str],
+ on_date, origin_session_id,
+ ) -> None:
+ today = date_type.today().isoformat()
+ on_date_str = (
+ on_date.isoformat() if isinstance(on_date, date_type)
+ else (on_date or today)
+ )
+ metadata: dict = {
+ "title": name,
+ "description": description,
+ "category": "event",
+ "status": "active",
+ "tags": tags,
+ "topics": topics,
+ "created": on_date_str,
+ "updated": today,
+ }
+ if origin_session_id:
+ metadata["originSessionId"] = origin_session_id
+
+ try:
+ Event.model_validate(metadata)
+ except ValidationError as e:
+ self._set_error({
+ "error": "Event schema validation failed",
+ "details": e.errors(include_context=False, include_url=False),
+ })
+ return
+
+ graph = self.file_store
+ conflicts = graph.collisions_after_create(target)
+ if conflicts:
+ taken = {Path(p).stem for p in graph.nodes}
+ suggested_name = next_suffixed_stem(taken, name)
+ self._set_error({
+ "error": (
+ f"stem `[[{name}]]` would resolve ambiguously "
+ f"to {len(conflicts) + 1} paths after this create"
+ ),
+ "conflicts": conflicts,
+ "suggested_name": suggested_name,
+ "hint": (
+ f"retry with name='{suggested_name}', or pick a "
+ f"semantic qualifier (e.g. '{name}-followup')."
+ ),
+ })
+ return
+
+ material_filenames = [m["filename"] for m in materials]
+ index_body = self._emit_body(content, material_filenames)
+ ok, payload = write_create(
+ self.file_store, target,
+ metadata=metadata, content=index_body,
+ )
+ if not ok:
+ self._set_error({
+ "path": str(target.resolve()),
+ "error": payload.get("error", "create failed"),
+ "details": payload,
+ })
+ return
+
+ material_paths: list[str] = []
+ for m in materials:
+ material_path = target.parent / m["filename"]
+ try:
+ material_path.write_text(m["content"], encoding="utf-8")
+ except Exception as e:
+ self.logger.warning(
+ f"sync: failed to write material {m['filename']}: {e}",
+ )
+ continue
+ material_paths.append(str(material_path.resolve()))
+
+ self._set_ok({
+ "path": str(target.resolve()),
+ "category": "event",
+ "status": "active",
+ "topics": topics,
+ "materials": material_paths,
+ "created": True,
+ "action": "created",
+ })
+
+ async def _append(
+ self, target: Path, content: str, materials: list[dict],
+ topics: list[str], tags: list[str],
+ ) -> None:
+ # Read current frontmatter + body.
+ try:
+ raw = target.read_text(encoding="utf-8")
+ except Exception as e:
+ self._set_error({"path": str(target.resolve()), "error": f"read failed: {e}"})
+ return
+ post = frontmatter.loads(raw)
+ meta = dict(post.metadata)
+
+ status = meta.get("status")
+ if status != "active":
+ # Don't extend a distilled / archived thread — make the agent pick
+ # a new name so the prior cognition isn't silently mutated.
+ graph = self.file_store
+ taken = {Path(p).stem for p in graph.nodes}
+ base = target.stem
+ suggested = next_suffixed_stem(taken, base)
+ self._set_error({
+ "path": str(target.resolve()),
+ "error": (
+ f"event `{base}` already exists with status={status!r}; "
+ f"pick a new name to start a fresh thread"
+ ),
+ "status": status,
+ "suggested_name": suggested,
+ })
+ return
+
+ folder = target.parent
+ existing_filenames = self._list_existing_materials(folder, target.name)
+ existing_set = set(existing_filenames)
+
+ # Resolve filename collisions for new materials.
+ resolved: list[tuple[str, str]] = [] # (filename_on_disk, content)
+ all_taken = set(existing_set)
+ for m in materials:
+ fname = self._resolve_filename(all_taken, m["filename"])
+ all_taken.add(fname)
+ resolved.append((fname, m["content"]))
+
+ # Write new materials to disk.
+ new_material_paths: list[str] = []
+ for fname, mcontent in resolved:
+ material_path = folder / fname
+ try:
+ material_path.write_text(mcontent, encoding="utf-8")
+ except Exception as e:
+ self.logger.warning(f"sync: failed to write material {fname}: {e}")
+ continue
+ new_material_paths.append(str(material_path.resolve()))
+ existing_filenames.append(fname)
+
+ # Rebuild the index body: narrative (existing + optional new Update
+ # section) + Materials footer (regenerated from disk).
+ narrative = self._strip_materials_footer(post.content)
+ if content.strip():
+ ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+ update_section = f"## Update — {ts}\n\n{content.rstrip()}\n"
+ narrative = (
+ f"{narrative.rstrip()}\n\n{update_section}"
+ if narrative else update_section
+ )
+
+ all_filenames_sorted = sorted(set(existing_filenames))
+ new_body = self._emit_body(narrative, all_filenames_sorted)
+
+ # Update frontmatter: union topics/tags, bump updated.
+ meta["topics"] = self._union(list(meta.get("topics") or []), topics)
+ meta["tags"] = self._union(list(meta.get("tags") or []), tags)
+ meta["updated"] = date_type.today().isoformat()
+
+ try:
+ Event.model_validate(meta)
+ except ValidationError as e:
+ self._set_error({
+ "path": str(target.resolve()),
+ "error": "Event schema validation failed on append",
+ "details": e.errors(include_context=False, include_url=False),
+ })
+ return
+
+ new_post = frontmatter.Post(new_body, **meta)
+ try:
+ target.write_text(frontmatter.dumps(new_post), encoding="utf-8")
+ except Exception as e:
+ self._set_error({"path": str(target.resolve()), "error": f"write failed: {e}"})
+ return
+
+ self._set_ok({
+ "path": str(target.resolve()),
+ "category": "event",
+ "status": "active",
+ "topics": meta["topics"],
+ "materials": new_material_paths,
+ "created": False,
+ "action": "appended",
+ })
diff --git a/reme2/mcp/steps/topic_create.py b/reme2/mcp/steps/topic_create.py
new file mode 100644
index 00000000..50e34baa
--- /dev/null
+++ b/reme2/mcp/steps/topic_create.py
@@ -0,0 +1,142 @@
+"""topic_create — create a new topic under topics/{folder}/{name}.md.
+
+The only file-creation entry point for topics. Enforces:
+ - path template (topics/{folder}/{name}.md)
+ - folder topic identification (folder == name)
+ - judgment-category strong-confidence (via Topic.model_validator)
+ - wikilink uniqueness (creating this file must not make `[[name]]` ambiguous)
+
+Writes via `write_create` — the canonical L1 invariant gate that also
+performs the wikilink-uniqueness check. The Ingestor's R-M-W loop is
+reserved for content-driven flows; topic creation is path-driven, so
+the direct call is sufficient.
+"""
+
+import json
+from datetime import date
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from reme2.component import R
+from reme2.component.base_step import BaseStep
+from reme2.memory.memory_io import write_create
+from reme2.schema.vault import Topic
+from reme2.utils.vault_paths import next_suffixed_stem, topic_path
+
+
+@R.register("topic_create")
+class TopicCreate(BaseStep):
+ """Create a topic file with the standard frontmatter template."""
+
+ def __init__(self, vault_root: str = "", topics_dir: str = "topics", **kwargs):
+ super().__init__(**kwargs)
+ self.vault_root = vault_root
+ self.topics_dir = topics_dir
+
+ def _root(self) -> Path:
+ if self.vault_root:
+ return Path(self.vault_root)
+ watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[union-attr]
+ return Path(watcher.watch_path)
+
+ async def execute(self):
+ assert self.context is not None
+ folder: str = self.context.get("folder", "")
+ name: str = self.context.get("name", "")
+ category: str = self.context.get("category", "")
+ description: str = self.context.get("description", "") or ""
+ content: str = self.context.get("content", "") or ""
+ confidence = self.context.get("confidence")
+ market = self.context.get("market")
+ ticker = self.context.get("ticker")
+ tags: list[str] = self.context.get("tags") or []
+
+ assert folder, "folder is required"
+ assert name, "name is required"
+ assert category, "category is required"
+
+ target = topic_path(self._root(), folder, name, self.topics_dir)
+ if target.exists():
+ self.context.response.success = False
+ self.context.response.answer = json.dumps({
+ "path": str(target.resolve()),
+ "error": "topic already exists; use memory_update / memory_property_update to modify",
+ }, ensure_ascii=False)
+ return
+
+ today = date.today().isoformat()
+ metadata: dict = {
+ "title": name,
+ "description": description,
+ "category": category,
+ "tags": tags,
+ "created": today,
+ "updated": today,
+ }
+ if confidence is not None:
+ metadata["confidence"] = confidence
+ if market is not None:
+ metadata["market"] = market
+ if ticker is not None:
+ metadata["ticker"] = ticker
+
+ # Topic schema validation (judgment categories require confidence).
+ try:
+ Topic.model_validate(metadata)
+ except ValidationError as e:
+ self.context.response.success = False
+ self.context.response.answer = json.dumps({
+ "error": "Topic schema validation failed",
+ "details": e.errors(include_context=False, include_url=False),
+ }, ensure_ascii=False)
+ return
+
+ # Pre-check uniqueness so we can surface a `suggested_name` to the
+ # agent. The actual write also routes through the same gate inside
+ # write_create, so this is a UX nicety, not a correctness check.
+ graph = self.file_store
+ conflicts = graph.collisions_after_create(target)
+ if conflicts:
+ taken = {Path(p).stem for p in graph.nodes}
+ suggested_name = next_suffixed_stem(taken, name)
+ self.context.response.success = False
+ self.context.response.answer = json.dumps({
+ "error": (
+ f"stem `[[{name}]]` would resolve ambiguously "
+ f"to {len(conflicts) + 1} paths after this create"
+ ),
+ "conflicts": conflicts,
+ "suggested_name": suggested_name,
+ "hint": (
+ f"retry with name='{suggested_name}' (numeric suffix), "
+ f"OR use a domain-specific qualifier (e.g. '{name}-Inc' / "
+ f"'{name}-v2') — semantic names beat numeric. If you "
+ f"actually meant the existing topic, call memory_get on "
+ f"one of `conflicts` instead."
+ ),
+ }, ensure_ascii=False)
+ return
+
+ ok, payload = write_create(
+ self.file_store, target,
+ metadata=metadata, content=content,
+ )
+
+ if not ok:
+ self.context.response.success = False
+ self.context.response.answer = json.dumps({
+ "path": str(target.resolve()),
+ "error": payload.get("error", "create failed"),
+ "details": payload,
+ }, ensure_ascii=False)
+ return
+
+ is_folder_topic = folder == name
+ self.context.response.success = True
+ self.context.response.answer = json.dumps({
+ "path": str(target.resolve()),
+ "category": category,
+ "is_folder_topic": is_folder_topic,
+ "created": True,
+ }, ensure_ascii=False)
diff --git a/reme2/memory/__init__.py b/reme2/memory/__init__.py
new file mode 100644
index 00000000..f1d48797
--- /dev/null
+++ b/reme2/memory/__init__.py
@@ -0,0 +1,22 @@
+"""Memory subsystem — the three services on top of the core engine.
+
+Per the architecture blueprint:
+
+ - retriever.py Read service. V + K + Graph BFS fusion + intent
+ routing. Registered as a Step (`backend: hybrid`);
+ MCP step shells in `reme2.mcp.steps.memory_retriever`
+ instantiate it on demand.
+ - ingestor.py Cold-write service. LLM-driven R-M-W curator.
+ Triggered on explicit handoff (task end / SessionEnd).
+ - maintainer.py Treatment service. Background Merge / Split / Decay /
+ Lint, woken by cron or thresholds.
+ - summarizer.py Auxiliary used by the services.
+
+Hot-write MCP step shells (sync, topic_create, memory_*) live in
+`reme2.mcp.steps`, NOT here — they bypass services and write MFS
+directly. Importing this package triggers @R.register on the three
+services so configs that name them resolve at boot.
+"""
+
+from . import retriever # noqa: F401 -- runs @R.register("hybrid")
+from . import maintainer # noqa: F401 -- runs @R.register("maintainer")
diff --git a/reme2/memory/ingestor.py b/reme2/memory/ingestor.py
new file mode 100644
index 00000000..9dfc71a5
--- /dev/null
+++ b/reme2/memory/ingestor.py
@@ -0,0 +1,187 @@
+"""Smart Ingestor — ReAct agent over the Memory File System.
+
+Per `structure.md` L31-44, the Ingestor is the SSOT engine: the **single
+write entry point** to the markdown vault. Every mutation (create, body
+edit, frontmatter flip, rename, delete, archive) flows through here.
+
+Mirrors `Summarizer`'s pattern — drives a `ReActAgent` whose toolkit is
+built by `memory_io.build_memory_toolkit`. The agent runs its own R-M-W
+loop: read related files via tools, decide which ones to mutate, call
+the right write tool. Every write tool records into an audit list, so
+the caller gets a deterministic mutation trail regardless of how the
+agent's reasoning unfolded.
+
+When no LLM is configured, falls back to a direct create from
+`target_path` + `metadata` + `content`. Edits/renames/deletes are not
+available without an LLM.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import zoneinfo
+from pathlib import Path
+
+from agentscope.agent import ReActAgent
+from agentscope.message import Msg
+from agentscope.tool import Toolkit
+from pydantic import BaseModel, Field
+
+from ..component.runtime_response import _set_answer, _to_jsonable
+from . import memory_io
+from .memory_io import MemoryIO, write_create
+from ..component import R
+from ..component.base_step import BaseStep
+from ..enumeration import ComponentEnum
+from ..utils.wikilink import extract_wikilinks
+
+
+class IngestResult(BaseModel):
+ """Audit trail for a single ingest call."""
+
+ applied: list[dict] = Field(default_factory=list, description="Successful ops with paths.")
+ rejected: list[dict] = Field(default_factory=list, description="Ops the validator refused.")
+ failed: list[dict] = Field(default_factory=list, description="Ops that errored at apply time.")
+ skipped: bool = Field(default=False, description="True if the LLM returned a SkipOp.")
+ used_llm: bool = Field(default=False, description="False = degraded path (no LLM configured).")
+
+ @property
+ def success(self) -> bool:
+ """Skipped or any-applied with no failures = success."""
+ if self.skipped:
+ return True
+ return len(self.applied) > 0 and len(self.failed) == 0
+
+
+@R.register("ingestor")
+class Ingestor(BaseStep):
+ """R-M-W ingestor exposed as a single-write-entry MCP tool.
+
+ Inputs (read from RuntimeContext):
+ content (str, required): the material being ingested.
+ hint (str, optional): caller guidance to the LLM.
+ target_path (str, optional): suggested file path; required for
+ the degraded path (no LLM).
+ metadata (dict, optional): suggested frontmatter; used by
+ the degraded path or as a hint to the LLM.
+ related_paths (list[str], optional): explicit related files,
+ auto-extended with wikilinks parsed from `content`.
+
+ Output (written to context.response.answer):
+ IngestResult JSON — applied/failed lists plus used_llm flag and
+ the agent's final-message summary.
+ """
+
+ def __init__(
+ self,
+ toolkit: Toolkit | None = None,
+ console_enabled: bool = False,
+ timezone: str | None = None,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.toolkit = toolkit
+ self.console_enabled = console_enabled
+ self.timezone = timezone
+
+ def _now(self) -> datetime.datetime:
+ if self.timezone:
+ try:
+ return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone))
+ except Exception as e:
+ self.logger.error(f"Invalid timezone: {self.timezone}, error={e}")
+ return datetime.datetime.now()
+
+ def _vault_root(self) -> Path:
+ watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
+ if watcher is None:
+ vr = getattr(self.file_store, "vault_root", None)
+ return Path(vr).resolve() if vr else Path.cwd().resolve()
+ return Path(getattr(watcher, "watch_path", ".")).resolve()
+
+ async def execute(self):
+ assert self.context is not None
+ content: str = self.context.get("content", "") or ""
+ hint: str = self.context.get("hint", "") or ""
+ target_path: str = self.context.get("target_path") or ""
+ metadata: dict = dict(self.context.get("metadata") or {})
+ related_paths: list[str] = list(self.context.get("related_paths") or [])
+
+ assert content, "content is required"
+
+ # Auto-discover wikilink targets in content as a hint for the agent.
+ for link in extract_wikilinks(content):
+ hit = memory_io.wikilink_lookup(self.file_store, link)["path"]
+ if hit and hit not in related_paths:
+ related_paths.append(hit)
+
+ as_llm = self._get_component_optional(ComponentEnum.AS_LLM, "default", "model")
+ if as_llm is None:
+ result = self._degraded(target_path, metadata, content)
+ self.context.response.success = result.success
+ _set_answer(self.context, result.model_dump())
+ return
+
+ vault_root = self._vault_root()
+ mio = MemoryIO(self.file_store, vault_root)
+ toolkit = mio.register_all(self.toolkit)
+
+ agent = ReActAgent(
+ name="reme_ingestor",
+ model=self.as_llm,
+ sys_prompt=self.prompt_format("system_prompt", vault_root=str(vault_root)),
+ formatter=self.as_llm_formatter,
+ toolkit=toolkit,
+ )
+ agent.set_console_output_enabled(self.console_enabled)
+
+ user_message: str = self.prompt_format(
+ "user_message",
+ today=self._now().strftime("%Y-%m-%d"),
+ vault_root=str(vault_root),
+ hint=hint or "(none)",
+ target_path=target_path or "(none)",
+ metadata=json.dumps(_to_jsonable(metadata), ensure_ascii=False),
+ related=json.dumps(related_paths, ensure_ascii=False),
+ content=content,
+ )
+
+ final_msg: Msg = await agent.reply(
+ Msg(name="reme", role="user", content=user_message),
+ )
+ summary = final_msg.get_text_content() or ""
+
+ result = IngestResult(used_llm=True)
+ for entry in mio.audit:
+ (result.applied if entry.get("ok") else result.failed).append(entry)
+ if not mio.audit and summary.strip().upper().startswith("SKIP"):
+ result.skipped = True
+
+ self.context.response.success = result.success
+ payload = result.model_dump()
+ payload["agent_summary"] = summary
+ _set_answer(self.context, payload)
+
+ def _degraded(self, target_path: str, metadata: dict, content: str) -> IngestResult:
+ """Without an LLM, only direct create from explicit target_path is
+ supported. Useful for tests and bootstrap scripts."""
+ result = IngestResult(used_llm=False)
+ if not target_path:
+ result.failed.append({
+ "op": "create",
+ "ok": False,
+ "error": "degraded path: target_path is required when no LLM is configured",
+ })
+ return result
+ path = Path(target_path)
+ if not path.is_absolute():
+ path = self._vault_root() / path
+ ok, payload = write_create(
+ self.file_store, path,
+ metadata=metadata, content=content,
+ )
+ bucket = result.applied if ok else result.failed
+ bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload})
+ return result
+
diff --git a/reme2/memory/ingestor.yaml b/reme2/memory/ingestor.yaml
new file mode 100644
index 00000000..33f37c12
--- /dev/null
+++ b/reme2/memory/ingestor.yaml
@@ -0,0 +1,110 @@
+system_prompt: |
+ You are the memory curator for a markdown vault — the LLM-driven
+ R-M-W loop over EXISTING files. You only run when the agent
+ EXPLICITLY HANDS OFF: at task completion, at session end, or when
+ the agent decides the working set is ready to be distilled. By that
+ point the agent has typically called `sync` along the way to
+ land raw facts as event folders. Your job: take whatever materials
+ the agent feeds you (inline text and/or paths), read them + any
+ related topics, decide which existing topics need updates, which
+ deserve a new topic, then flip each distilled event's status.
+
+ vault_root: {vault_root}
+
+ # Scope of THIS tool
+ - You are the COLD path. You DO NOT run on every turn — only on
+ explicit handoff (task done / session ending / agent invokes you
+ directly). The hot path (`sync`, deterministic, no LLM)
+ has already preserved raw facts continuously through the task.
+ - The agent supplies the working set in two interchangeable forms:
+ * `content` — inline material the agent is giving you directly
+ (a distillation hint, a session summary, raw text it wants
+ folded into the graph).
+ * `related_paths` — paths the agent points you at (event folder
+ indexes, individual material files, candidate topics it flagged
+ for update). Both forms can appear together; treat them as a
+ single working set.
+ - An event is a FOLDER containing the index `{name}.md` plus
+ materials (raw conversation snippets, tool outputs, data dumps).
+ Whenever a path in `related_paths` is an event index, `memory_get`
+ the index first, then `memory_get` any materials whose content you
+ need (the index lists them under `## Materials`).
+
+ # Vault conventions
+ - Topics live under `topics/{{folder}}/{{name}}.md`. A folder topic
+ has folder == name.
+ - Events live under `events/{{YYYY-MM-DD}}/{{name}}/{{name}}.md` —
+ this is an INDEX inside a folder; sibling files are materials.
+ - Frontmatter is YAML; `category`, `created`, `updated`, `tags`,
+ `status` are common.
+ - Cross-file references use `[[wikilink]]` syntax (stem-form `[[X]]`
+ or path-form `[[topics/X/X]]`).
+
+ # Available tools
+ Read tools (use these to gather context BEFORE writing):
+ - memory_get(path, include_chunks=False): full file content + frontmatter.
+ Call this on each event index AND on the materials it lists.
+ - memory_list(path_prefix=None, tags=None, metadata=None, limit=100):
+ list indexed files filtered by prefix / tags / frontmatter.
+ - memory_resolve_wikilink(wikilink): resolve `[[X]]` to a path.
+ - memory_backlinks(path): files linking to a given path.
+ - memory_links(path): files a given path links to.
+
+ Write tools (each returns success + payload; mutations are SSOT-routed):
+ - memory_update(path, old_string, new_string, replace_all=False):
+ body edit by exact-string substitution. Use a tail snippet to append.
+ - memory_property_update(path, key, value): change one frontmatter
+ key (value=null deletes it). After distilling an event into one or
+ more topics, flip that event's status to "distilled" with this.
+ - memory_create(path, metadata, content, overwrite=False, force=False):
+ new file. Reserve for genuinely NEW topics — do NOT use this to log
+ events; `sync` (deterministic) owns events. ALL paths must
+ be ABSOLUTE under vault_root.
+ - memory_rename(old_path, new_path): move file + rewrite cross-vault
+ wikilinks.
+ - memory_delete(path): remove a file.
+ - memory_archive(path): flip `status: archived` and move under
+ `/Archive/`.
+
+ # Decision rules
+ 1. If material is ALREADY covered by existing topics → reply with a
+ single line `SKIP: ` and call no tools.
+ 2. If material CONTRADICTS an existing block → memory_update with a
+ unique snippet of the outdated text and the corrected replacement.
+ 3. If material EXTENDS an existing topic → memory_update using a
+ unique TAIL snippet of the existing body, with new_string =
+ tail + blank line + new content.
+ 4. If material warrants a GENUINELY NEW topic → memory_create at
+ `topics/{{folder}}/{{name}}.md`. Do NOT memory_create under events/.
+ 5. After integrating an event's content into a topic, flip that event's
+ status to "distilled" with memory_property_update.
+ 6. Never delete unless the material explicitly asks for deletion.
+ 7. Keep edits minimal — read related files first, edit only what
+ must change.
+ 8. Always include reasonable frontmatter on memory_create — at minimum
+ `title`, `category`, `created`, `updated`. Use today's date for
+ `created` / `updated`.
+ 9. After all writes, end with a one-paragraph summary of what you did
+ and why.
+
+user_message: |
+ # CONTEXT
+ today: {today}
+ vault_root: {vault_root}
+
+ # WORKING SET (handed off by the agent)
+ caller hint: {hint}
+ target_path hint: {target_path}
+ metadata hint: {metadata}
+ related paths (auto-discovered from wikilinks in content + caller's related_paths): {related}
+
+ content (inline material the agent is feeding you — distillation
+ hint, session summary, or raw text):
+ {content}
+
+ Treat `content` (inline) and `related paths` (pointers) as a single
+ working set. For each related path that's an event index, read the
+ index then read any materials it lists. Inspect the related topics,
+ then perform the minimal set of writes needed to distill this
+ material into the topic graph. Flip any distilled event's status as
+ part of the same call. End with a one-paragraph summary.
diff --git a/reme2/memory/maintainer.py b/reme2/memory/maintainer.py
new file mode 100644
index 00000000..4776d225
--- /dev/null
+++ b/reme2/memory/maintainer.py
@@ -0,0 +1,511 @@
+"""Maintainer — treatment service. One pass, four signals, one plan.
+
+Per the architecture blueprint, the Maintainer is the third memory
+service alongside the Retriever (read) and the Ingestor (cold-write).
+It's woken by cron or thresholds, not by per-turn agent calls.
+
+DESIGN: single `Maintainer` Step, NOT four. Vault hygiene is one
+operation with four signal sources that contend for the same files —
+splitting them into independent Steps would let merge & split reverse
+each other, let decay archive a file that merge wanted to absorb, and
+force every cron tick to scan the vault four times. So the Maintainer
+follows a plan-then-apply pipeline:
+
+ scan_signals() # one walk, all signals shared
+ ↓
+ propose_*() # each signal source emits Op records
+ ↓
+ resolve_conflicts() # data-driven matrix dedupes / orders ops
+ ↓
+ apply() # lint → decay → merge → split, dry-run aware
+ ↓
+ audit # unified trail returned to caller
+
+OPERATIONS
+
+ LintFinding read-only diagnostic; never conflicts.
+ DecayOp move a stale event under //.
+ MergeOp absorb sources[] into canonical; rewrite incoming
+ wikilinks; archive sources.
+ SplitOp extract sections of source[] into new files; replace
+ in original with [[…]] stubs.
+
+CONFLICT MATRIX (resolved before apply)
+
+ MergeOp(source=P) ⊕ SplitOp(source=P) → drop split
+ MergeOp(source=P) ⊕ DecayOp(path=P) → drop decay
+ SplitOp(source=P) ⊕ DecayOp(path=P) → drop split
+ MergeOp(canonical=A,…) × N (same A) → union sources
+ SplitOp(source=P) × N → keep highest confidence
+ LintFinding ⊕ anything → coexist
+
+APPLY ORDER is fixed: lint → decay → merge → split. This guarantees
+that by the time split runs, merge has already rewritten paths; if a
+split target was absorbed by a merge, apply skips it and records a
+`stale_target` audit entry instead of crashing.
+
+LLM-DRIVEN PROPOSERS (merge/split) are scaffolded — the structure +
+op shape + conflict path are wired and verified, but the LLM-backed
+similarity / split heuristics are pending design and currently return
+empty proposal lists. Lint and decay are fully implemented.
+"""
+
+from __future__ import annotations
+
+import datetime
+from collections import defaultdict
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ..component import R
+from ..component.base_step import BaseStep
+from ..component.runtime_response import _set_answer
+from ..enumeration import ComponentEnum
+from ..schema.vault.registry import schema_for
+from . import memory_io
+
+
+# ---------------------------------------------------------------------------
+# Op records
+# ---------------------------------------------------------------------------
+
+
+class LintFinding(BaseModel):
+ """Read-only diagnostic. Never conflicts; never mutates."""
+
+ op: Literal["lint"] = "lint"
+ path: str
+ kind: Literal["broken_wikilink", "schema_violation", "stem_collision"]
+ detail: str
+
+
+class DecayOp(BaseModel):
+ """Archive a stale event. Moves the file; doesn't change body."""
+
+ op: Literal["decay"] = "decay"
+ path: str
+ age_days: int
+ reason: str = "past freshness window"
+
+
+class MergeOp(BaseModel):
+ """Consolidate `sources` into `canonical`. Sources get archived."""
+
+ op: Literal["merge"] = "merge"
+ canonical: str
+ sources: list[str] = Field(default_factory=list)
+ confidence: float = 0.0
+ reason: str = ""
+
+
+class SplitOp(BaseModel):
+ """Extract sections of `source` into new sibling files."""
+
+ op: Literal["split"] = "split"
+ source: str
+ sections: list[dict] = Field(default_factory=list) # [{title, body, target_path}]
+ confidence: float = 0.0
+ reason: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Signals: one scan, all observers consume the same dict
+# ---------------------------------------------------------------------------
+
+
+class FileSignal(BaseModel):
+ """Per-file derived signals shared by every proposer.
+
+ Built once by `_scan_signals`. Holds only what's cheap to compute
+ from `file_store.nodes` + edge index — no body reads, no LLM calls.
+ Heavier signals (token counts, embeddings) are pulled lazily inside
+ the proposers that actually need them.
+ """
+
+ path: str
+ relpath: str = "" # path relative to vault_root, "" if outside
+ category: str = ""
+ status: str = ""
+ age_days: int = 0
+ metadata: dict = Field(default_factory=dict)
+ declared_topics: list[str] = Field(default_factory=list) # raw [[…]] strings
+
+
+# ---------------------------------------------------------------------------
+# Maintainer
+# ---------------------------------------------------------------------------
+
+
+@R.register("maintainer")
+class Maintainer(BaseStep):
+ """Unified vault hygiene. Scan → propose → resolve → apply.
+
+ Reads from RuntimeContext (all optional):
+ ops (list[str], default ["lint","decay"]):
+ subset of {"lint","decay","merge","split"} to run.
+ Merge/split require an LLM and are off by default.
+ dry_run (bool, default True): if True, returns the plan
+ but doesn't mutate the vault.
+ decay_days (int, default constructor `decay_days`): freshness
+ window for the decay proposer.
+ target_prefix (str, default ""): restrict scan to relpaths
+ starting with this prefix (e.g. "events/").
+ token_threshold (int, default constructor): split threshold.
+ merge_threshold (float, default constructor): cluster cutoff.
+
+ Writes to ctx.response.answer:
+ {
+ "ops_run": [...], # which proposers ran
+ "scanned": int,
+ "proposed": [op,...], # raw, before conflict resolution
+ "plan": [op,...], # after conflict resolution
+ "applied": [op,...], # successfully mutated
+ "skipped": [op,...], # dropped by conflict resolution
+ "failed": [op,...], # apply errored
+ "dry_run": bool,
+ "ran_at": iso,
+ }
+ """
+
+ # Fixed apply order — see module docstring CONFLICT MATRIX section.
+ _APPLY_ORDER = ("lint", "decay", "merge", "split")
+
+ def __init__(
+ self,
+ decay_days: int = 90,
+ archive_dir: str = "Archive",
+ target_status: str = "distilled",
+ token_threshold: int = 4000,
+ merge_threshold: float = 0.85,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.decay_days = decay_days
+ self.archive_dir = archive_dir
+ self.target_status = target_status
+ self.token_threshold = token_threshold
+ self.merge_threshold = merge_threshold
+
+ # -- entry point --------------------------------------------------------
+
+ async def execute(self):
+ assert self.context is not None
+ params = self._load_params()
+
+ signals = self._scan_signals(target_prefix=params["target_prefix"])
+
+ proposed: list[BaseModel] = []
+ if "lint" in params["ops"]:
+ proposed.extend(self._propose_lint(signals))
+ if "decay" in params["ops"]:
+ proposed.extend(self._propose_decay(signals, params["decay_days"]))
+ if "merge" in params["ops"]:
+ proposed.extend(await self._propose_merge(signals, params["merge_threshold"]))
+ if "split" in params["ops"]:
+ proposed.extend(await self._propose_split(signals, params["token_threshold"]))
+
+ plan, dropped = self._resolve_conflicts(proposed)
+
+ applied: list[dict] = []
+ failed: list[dict] = []
+ if not params["dry_run"]:
+ applied, failed = await self._apply(plan)
+
+ audit = {
+ "ops_run": params["ops"],
+ "scanned": len(signals),
+ "proposed": [_dump(o) for o in proposed],
+ "plan": [_dump(o) for o in plan],
+ "applied": applied,
+ "skipped": [_dump(o) for o in dropped],
+ "failed": failed,
+ "dry_run": params["dry_run"],
+ "ran_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
+ }
+ self.context.response.success = len(failed) == 0
+ _set_answer(self.context, audit)
+
+ # -- params -------------------------------------------------------------
+
+ def _load_params(self) -> dict:
+ """Pull RuntimeContext kwargs with proper defaults.
+
+ Centralizing this kills the `int(ctx.get(...))` / `float(ctx.get(...))`
+ pattern that bites when `.get` returns None — every conversion
+ below has a guaranteed-non-None operand.
+ """
+ ctx = self.context
+ assert ctx is not None
+ ops = ctx.get("ops") or ["lint", "decay"]
+ if not isinstance(ops, list):
+ raise ValueError(f"ops must be a list of strings, got {type(ops).__name__}")
+ bad = [o for o in ops if o not in self._APPLY_ORDER]
+ if bad:
+ raise ValueError(f"unknown ops: {bad}; allowed: {list(self._APPLY_ORDER)}")
+ return {
+ "ops": ops,
+ "dry_run": bool(ctx.get("dry_run", True)),
+ "decay_days": int(ctx.get("decay_days") or self.decay_days),
+ "target_prefix": str(ctx.get("target_prefix") or ""),
+ "token_threshold": int(ctx.get("token_threshold") or self.token_threshold),
+ "merge_threshold": float(ctx.get("merge_threshold") or self.merge_threshold),
+ }
+
+ # -- scan ---------------------------------------------------------------
+
+ def _vault_root(self) -> Path | None:
+ watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
+ if watcher is None:
+ return None
+ return Path(getattr(watcher, "watch_path", ".")).resolve()
+
+ def _scan_signals(self, *, target_prefix: str = "") -> list[FileSignal]:
+ """One walk over the indexed files. Cheap signals only."""
+ vault_root = self._vault_root()
+ now = datetime.datetime.now().timestamp()
+ signals: list[FileSignal] = []
+ for path, meta in memory_io.iter_files(self.file_store):
+ relpath = ""
+ if vault_root is not None:
+ try:
+ relpath = str(Path(path).resolve().relative_to(vault_root))
+ except ValueError:
+ pass
+ if target_prefix and not relpath.startswith(target_prefix):
+ continue
+ fm = meta.metadata or {}
+ age_seconds = max(0.0, now - (meta.st_mtime or now))
+ signals.append(FileSignal(
+ path=path,
+ relpath=relpath,
+ category=str(fm.get("category") or ""),
+ status=str(fm.get("status") or ""),
+ age_days=int(age_seconds // 86400),
+ metadata=fm,
+ declared_topics=list(fm.get("topics") or []),
+ ))
+ return signals
+
+ # -- proposers ----------------------------------------------------------
+
+ def _propose_lint(self, signals: list[FileSignal]) -> list[LintFinding]:
+ """Broken wikilinks + frontmatter schema violations + stem collisions."""
+ out: list[LintFinding] = []
+ for sig in signals:
+ for link in sig.declared_topics:
+ if not memory_io.wikilink_lookup(self.file_store, link)["exists"]:
+ out.append(LintFinding(
+ path=sig.path, kind="broken_wikilink",
+ detail=f"unresolved wikilink {link!r}",
+ ))
+ cls = schema_for(sig.category)
+ if cls is not None:
+ try:
+ cls(**sig.metadata)
+ except Exception as e:
+ out.append(LintFinding(
+ path=sig.path, kind="schema_violation",
+ detail=f"{cls.__name__}: {type(e).__name__}: {e}"[:240],
+ ))
+ # Stem collisions: the engine API exposes the ambiguous-stem map.
+ ambig = memory_io.all_ambiguous_wikilinks(self.file_store)
+ for stem, paths in ambig.items():
+ for p in paths:
+ out.append(LintFinding(
+ path=p, kind="stem_collision",
+ detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}",
+ ))
+ return out
+
+ def _propose_decay(
+ self, signals: list[FileSignal], decay_days: int,
+ ) -> list[DecayOp]:
+ """Distilled events past the freshness window."""
+ out: list[DecayOp] = []
+ for sig in signals:
+ if sig.category != "event":
+ continue
+ if sig.status != self.target_status:
+ continue
+ if sig.age_days < decay_days:
+ continue
+ out.append(DecayOp(
+ path=sig.path, age_days=sig.age_days,
+ reason=f"event {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
+ ))
+ return out
+
+ async def _propose_merge(
+ self, signals: list[FileSignal], threshold: float,
+ ) -> list[MergeOp]:
+ """Cluster near-duplicate topics → MergeOps (LLM-assisted).
+
+ SCAFFOLD: the structure is wired but the clustering implementation
+ is pending. Returns []. When implemented:
+ 1. Embed titles + descriptions; cluster on cosine ≥ threshold.
+ 2. For each cluster, ask the LLM to pick the canonical and
+ summarize what the merged body should preserve.
+ 3. Emit MergeOp(canonical, sources=[non-canonical], confidence,
+ reason=LLM justification).
+ """
+ return []
+
+ async def _propose_split(
+ self, signals: list[FileSignal], token_threshold: int,
+ ) -> list[SplitOp]:
+ """Topics over the token threshold → SplitOps (LLM-assisted).
+
+ SCAFFOLD: returns []. When implemented:
+ 1. Filter signals to topics whose body exceeds token_threshold.
+ 2. Read body, ask the LLM for a section partition + filenames.
+ 3. Emit SplitOp(source, sections=[{title,body,target_path}],
+ confidence, reason).
+ """
+ return []
+
+ # -- conflict resolution ------------------------------------------------
+
+ def _resolve_conflicts(
+ self, proposed: list[BaseModel],
+ ) -> tuple[list[BaseModel], list[BaseModel]]:
+ """Apply the conflict matrix; return (kept_plan, dropped).
+
+ Steps:
+ 1. Union MergeOps that share a canonical.
+ 2. Dedupe SplitOps by source (highest confidence wins).
+ 3. For each path, resolve {merge-source, split-source, decay}
+ contention per the matrix in the module docstring.
+ 4. Lint findings are pass-through.
+ """
+ lints = [o for o in proposed if isinstance(o, LintFinding)]
+ merges = [o for o in proposed if isinstance(o, MergeOp)]
+ splits = [o for o in proposed if isinstance(o, SplitOp)]
+ decays = [o for o in proposed if isinstance(o, DecayOp)]
+
+ dropped: list[BaseModel] = []
+
+ # (1) Union merges by canonical.
+ merged_by_canon: dict[str, MergeOp] = {}
+ for m in merges:
+ existing = merged_by_canon.get(m.canonical)
+ if existing is None:
+ merged_by_canon[m.canonical] = m
+ else:
+ merged_sources = list(dict.fromkeys(existing.sources + m.sources))
+ existing.sources = merged_sources
+ existing.confidence = max(existing.confidence, m.confidence)
+ existing.reason = (existing.reason + "; " + m.reason).strip("; ")
+ merges = list(merged_by_canon.values())
+
+ # (2) Dedupe splits by source.
+ split_by_source: dict[str, SplitOp] = {}
+ for s in splits:
+ current = split_by_source.get(s.source)
+ if current is None or s.confidence > current.confidence:
+ if current is not None:
+ dropped.append(current)
+ split_by_source[s.source] = s
+ else:
+ dropped.append(s)
+ splits = list(split_by_source.values())
+
+ # (3) Build path → owning op index.
+ merge_paths: set[str] = set()
+ for m in merges:
+ merge_paths.update(m.sources)
+ merge_paths.add(m.canonical)
+
+ kept_splits: list[SplitOp] = []
+ for s in splits:
+ if s.source in merge_paths:
+ dropped.append(s) # MergeOp ⊕ SplitOp(source=P) → drop split
+ else:
+ kept_splits.append(s)
+
+ split_sources = {s.source for s in kept_splits}
+
+ kept_decays: list[DecayOp] = []
+ for d in decays:
+ if d.path in merge_paths:
+ dropped.append(d) # MergeOp ⊕ DecayOp(P) → drop decay
+ elif d.path in split_sources:
+ # SplitOp(source=P) ⊕ DecayOp(P) → matrix says drop split,
+ # but split was kept above (no merge contention) so the
+ # decay wins here: archive trumps refining what's about
+ # to leave the active set.
+ # → drop the split, keep decay.
+ for s in list(kept_splits):
+ if s.source == d.path:
+ kept_splits.remove(s)
+ dropped.append(s)
+ kept_decays.append(d)
+ else:
+ kept_decays.append(d)
+
+ plan: list[BaseModel] = []
+ plan.extend(lints)
+ plan.extend(kept_decays)
+ plan.extend(merges)
+ plan.extend(kept_splits)
+ return plan, dropped
+
+ # -- apply --------------------------------------------------------------
+
+ async def _apply(
+ self, plan: list[BaseModel],
+ ) -> tuple[list[dict], list[dict]]:
+ """Execute the plan in fixed order. Each phase yields audit dicts.
+
+ Lint never mutates → recorded as applied no-op so the audit shows
+ which findings the cron run surfaced.
+
+ Decay / Merge / Split call into the existing memory_io primitives
+ once the heuristics land; for now they record `pending_apply` so
+ the dry_run=False path still produces a stable audit shape.
+ """
+ applied: list[dict] = []
+ failed: list[dict] = []
+
+ # Group ops by type, apply in fixed order.
+ by_type: dict[str, list] = defaultdict(list)
+ for o in plan:
+ by_type[o.op].append(o) # type: ignore[attr-defined]
+
+ for kind in self._APPLY_ORDER:
+ for op in by_type.get(kind, []):
+ try:
+ result = await self._apply_one(op)
+ applied.append({"op": op.op, **result}) # type: ignore[attr-defined]
+ except Exception as e:
+ failed.append({
+ "op": op.op, # type: ignore[attr-defined]
+ "payload": _dump(op),
+ "error": f"{type(e).__name__}: {e}",
+ })
+ return applied, failed
+
+ async def _apply_one(self, op: BaseModel) -> dict:
+ if isinstance(op, LintFinding):
+ # Diagnostic-only: surfacing the finding IS the work.
+ return {"path": op.path, "kind": op.kind, "noop": True}
+ if isinstance(op, DecayOp):
+ # TODO: wire to MemoryArchive once we're ready to actually
+ # move files from a cron context. For now, surface intent.
+ return {"path": op.path, "status": "pending_apply",
+ "would": "flip status=archived + move to archive_dir"}
+ if isinstance(op, MergeOp):
+ return {"canonical": op.canonical, "sources": op.sources,
+ "status": "pending_apply",
+ "would": "merge bodies + rewrite incoming wikilinks + archive sources"}
+ if isinstance(op, SplitOp):
+ return {"source": op.source, "sections": len(op.sections),
+ "status": "pending_apply",
+ "would": "extract sections + replace with [[…]] stubs"}
+ raise TypeError(f"unknown op type: {type(op).__name__}")
+
+
+def _dump(op: BaseModel) -> dict:
+ """JSON-friendly snapshot of an op record."""
+ return op.model_dump()
diff --git a/reme2/memory/memory_io.py b/reme2/memory/memory_io.py
new file mode 100644
index 00000000..ed89b019
--- /dev/null
+++ b/reme2/memory/memory_io.py
@@ -0,0 +1,849 @@
+"""Memory File System engine API — the core engine's outward surface.
+
+The .md files are the SSOT (per `structure.md` §"核心引擎"). The engine
+is layered:
+
+ Memory File System → Watcher & Parser → Projections (vector / FTS / graph)
+ (write entry) (incremental) (read entry, derived)
+
+This module is the **single public API surface** over that engine. Every
+consumer — MCP step shells, the three memory services (Retriever,
+Ingestor, Maintainer), and the agent toolkit — talks to the engine
+through these functions, not by reaching into `BaseFileStore` directly.
+That keeps `file_store` an implementation detail (could be local sqlite,
+remote, etc.) and gives the layering one place to evolve.
+
+Four sections:
+
+ 1. CRUD writes — write_create / delete / update / property_update
+ / rename / archive. The MFS write entry.
+ 2. MFS reads — read_file / list_files / links_of / backlinks_of
+ / wikilink_lookup / count_tokens / iter_files.
+ Primary-key lookups against the file_store cache.
+ 3. Projections — vector_search / keyword_search / expand_neighbors
+ / extract_anchors / chunks_by_paths / make_chunk_filter
+ / all_ambiguous_wikilinks. The read entry to derived
+ indexes (composed by Retriever into V+K+graph fusion).
+ 4. Toolkit — `MemoryIO` class wrapping the read/write helpers as
+ `agentscope.tool` callables for ReActAgent (Ingestor).
+
+Lives in `reme2/memory/` (not `reme2/mcp/`) so memory services and the
+MCP transport layer can both consume it without forming an import cycle
+through the transport layer.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Any
+
+import frontmatter
+from agentscope.message import TextBlock
+from agentscope.tool import Toolkit, ToolResponse
+
+from ..component.runtime_response import _to_jsonable
+from ..schema import ChunkFilter, FileChunk, FileMetadata
+from ..utils.wikilink import WIKILINK_RE
+
+
+# ===========================================================================
+# Section 1 — CRUD writes
+# ===========================================================================
+#
+# Every mutation in the system funnels through these. Hot-write MCP shells
+# (sync, topic_create, memory_*) call them directly; cold-write services
+# (Ingestor R-M-W, Maintainer decay) compose them.
+
+
+def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
+ """Rewrite wikilink targets in raw text.
+
+ Only the `target` portion of `[[target]]` / `[[target#anchor]]` /
+ `[[target|alias]]` / `![[target]]` is replaced; anchors, aliases,
+ and embed prefixes are preserved.
+ """
+ if not mapping:
+ return text
+
+ def sub(m: re.Match) -> str:
+ target_raw = m.group(1)
+ target = target_raw.strip()
+ if target in mapping:
+ return m.group(0).replace(target_raw, mapping[target], 1)
+ return m.group(0)
+
+ return WIKILINK_RE.sub(sub, text)
+
+
+def write_create(
+ file_store,
+ path: Path,
+ metadata: dict,
+ content: str,
+ overwrite: bool = False,
+ force: bool = False,
+) -> tuple[bool, dict]:
+ """Single L1 entry point for creating a markdown file.
+
+ All file creation in the project must funnel through this so the
+ wikilink uniqueness invariant is enforced in exactly one place.
+
+ Refuses (returns (False, payload)) when:
+ - file already exists (unless overwrite=True)
+ - creating it would make `[[stem]]` resolve ambiguously against
+ the current file_store (unless force=True)
+ """
+ if path.exists() and not overwrite:
+ return False, {"path": str(path), "error": "file already exists"}
+
+ if not force:
+ conflicts = file_store.collisions_after_create(path)
+ if conflicts:
+ return False, {
+ "path": str(path),
+ "error": (
+ f"stem `[[{path.stem}]]` would resolve ambiguously "
+ f"to {len(conflicts) + 1} paths after this create"
+ ),
+ "conflicts": conflicts,
+ "hint": (
+ f"either rename to a unique stem, or have callers "
+ f"link via the explicit-path form "
+ f"`[[{path.parent.name}/{path.stem}]]`; pass "
+ f"force=true only if you accept the ambiguity"
+ ),
+ }
+
+ path.parent.mkdir(parents=True, exist_ok=True)
+ post = frontmatter.Post(content, **metadata)
+ path.write_text(frontmatter.dumps(post), encoding="utf-8")
+ return True, {"path": str(path), "created": True}
+
+
+def write_delete(path: Path | str) -> tuple[bool, dict]:
+ """Delete a file. Watcher removes from store + graph."""
+ target = Path(path)
+ if not target.exists():
+ return False, {"path": str(target), "error": "not found"}
+ target.unlink()
+ return True, {"path": str(target), "deleted": True}
+
+
+def write_update(
+ path: Path | str,
+ old_string: str,
+ new_string: str,
+ replace_all: bool = False,
+) -> tuple[bool, dict]:
+ """Edit-style content update — replace `old_string` with `new_string`."""
+ target = Path(path)
+ if not target.is_file():
+ return False, {"path": str(target), "error": "file not found"}
+ if not old_string:
+ return False, {
+ "path": str(target),
+ "error": "old_string is required (use write_create to write a new file)",
+ }
+ raw = target.read_text(encoding="utf-8")
+ occurrences = raw.count(old_string)
+ if occurrences == 0:
+ return False, {"path": str(target), "error": "old_string not found in file"}
+ if occurrences > 1 and not replace_all:
+ return False, {
+ "path": str(target),
+ "error": f"old_string appears {occurrences} times; pass replace_all=true to replace all",
+ "occurrences": occurrences,
+ }
+ if replace_all:
+ new_raw = raw.replace(old_string, new_string)
+ else:
+ new_raw = raw.replace(old_string, new_string, 1)
+ target.write_text(new_raw, encoding="utf-8")
+ return True, {
+ "path": str(target),
+ "replaced": occurrences if replace_all else 1,
+ }
+
+
+def write_property_update(path: Path | str, key: str, value) -> tuple[bool, dict]:
+ """Update a single YAML frontmatter key. value=None deletes the key."""
+ target = Path(path)
+ if not target.is_file():
+ return False, {"path": str(target), "error": "file not found"}
+ raw = target.read_text(encoding="utf-8")
+ post = frontmatter.loads(raw)
+ if value is None:
+ post.metadata.pop(key, None)
+ else:
+ post.metadata[key] = value
+ target.write_text(frontmatter.dumps(post), encoding="utf-8")
+ return True, {"path": str(target), "key": key, "value": value}
+
+
+def write_rename(
+ file_store,
+ vault_root: Path | str,
+ old_path: Path | str,
+ new_path: Path | str,
+) -> tuple[bool, dict]:
+ """Rename a file and rewrite incoming wikilinks across the vault.
+
+ Atomically moves `old_path` to `new_path`, then rewrites short-form
+ `[[old_stem]]` and path-form `[[old_relative]]` wikilinks in every
+ file that already had a *resolved* link to old_path. Look-up uses
+ `file_store.get_backlinks(old_path)` so the work is O(K) where K is
+ the number of incoming references — not a full vault scan.
+
+ Refuses if:
+ - `old_path` doesn't exist
+ - `new_path` already exists
+ - the rename would make `[[new_stem]]` resolve ambiguously
+ """
+ old_p = Path(old_path).resolve()
+ new_p = Path(new_path).resolve()
+
+ if not old_p.is_file():
+ return False, {"old_path": str(old_p), "error": "old_path not found"}
+ if new_p.exists():
+ return False, {"new_path": str(new_p), "error": "new_path already exists"}
+ if old_p == new_p:
+ return False, {"error": "old_path and new_path are the same"}
+
+ conflicts = file_store.collisions_after_create(new_p)
+ if conflicts:
+ return False, {
+ "error": (
+ f"stem `[[{new_p.stem}]]` would resolve ambiguously "
+ f"to {len(conflicts) + 1} paths after this rename"
+ ),
+ "conflicts": conflicts,
+ "hint": (
+ f"either rename to a unique stem (consider a "
+ f"domain-specific suffix), or have callers link via "
+ f"the explicit-path form `[[{new_p.parent.name}/{new_p.stem}]]`"
+ ),
+ }
+
+ vault_root_p = Path(vault_root).resolve()
+ old_stem = old_p.stem
+ new_stem = new_p.stem
+ replacements: dict[str, str] = {}
+ if old_stem != new_stem:
+ replacements[old_stem] = new_stem
+ try:
+ old_rel = str(old_p.relative_to(vault_root_p).with_suffix(""))
+ new_rel = str(new_p.relative_to(vault_root_p).with_suffix(""))
+ if old_rel != new_rel:
+ replacements[old_rel] = new_rel
+ replacements[old_rel + ".md"] = new_rel + ".md"
+ except ValueError:
+ pass # paths outside vault root — skip path-form rewrite
+
+ referring_paths = [m.path for m, _ in file_store.get_backlinks(str(old_p))]
+
+ new_p.parent.mkdir(parents=True, exist_ok=True)
+ old_p.rename(new_p)
+
+ updated_files: list[str] = []
+ write_errors: list[dict] = []
+ if replacements and referring_paths:
+ for path in referring_paths:
+ file_path = Path(path)
+ if not file_path.is_file():
+ continue
+ try:
+ raw = file_path.read_text(encoding="utf-8")
+ new_raw = _replace_wikilink_targets(raw, replacements)
+ if new_raw != raw:
+ file_path.write_text(new_raw, encoding="utf-8")
+ updated_files.append(path)
+ except Exception as exc:
+ write_errors.append({"path": path, "error": str(exc)})
+
+ return True, {
+ "old_path": str(old_p),
+ "new_path": str(new_p),
+ "stem_changed": old_stem != new_stem,
+ "replacements": replacements,
+ "referring_count": len(referring_paths),
+ "updated_files": updated_files,
+ "write_errors": write_errors,
+ }
+
+
+def write_archive(
+ vault_root: Path | str,
+ path: Path | str,
+ archive_dir_name: str = "Archive",
+) -> tuple[bool, dict]:
+ """Archive a file: flip `status: archived`, then move under `//`.
+
+ Composed by the Maintainer's decay pass when an event falls past its
+ freshness window. Backlinks are *not* rewritten — dangling links to
+ archived files are the intended audit trail.
+ """
+ src = Path(path).resolve()
+ if not src.is_file():
+ return False, {"path": str(src), "error": "file not found"}
+
+ vault = Path(vault_root).resolve()
+ try:
+ rel = src.relative_to(vault)
+ except ValueError:
+ return False, {
+ "path": str(src),
+ "error": f"path is outside vault_root {vault}",
+ }
+
+ dst = vault / archive_dir_name / rel
+ if dst.exists():
+ return False, {
+ "path": str(src),
+ "error": f"archive destination already exists: {dst}",
+ }
+
+ ok, prop_payload = write_property_update(src, "status", "archived")
+ if not ok:
+ return False, {**prop_payload, "stage": "property_update"}
+
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ shutil.move(str(src), str(dst))
+ except OSError as exc:
+ return False, {
+ "path": str(src),
+ "error": f"move failed: {exc}",
+ "stage": "move",
+ }
+ return True, {
+ "old_path": str(src),
+ "new_path": str(dst),
+ "archived": True,
+ }
+
+
+# ===========================================================================
+# Section 2 — MFS reads (primary-key lookups against the file_store cache)
+# ===========================================================================
+#
+# These don't touch the projection indexes — they hit the in-memory file
+# meta + edge cache that the file_store maintains as a mirror of disk.
+# For body / chunk content reads, `read_file` does fall through to the
+# disk to get the latest text (file_store cache may lag a write).
+
+
+async def read_file(
+ file_store,
+ path: str,
+ *,
+ include_chunks: bool = False,
+) -> dict:
+ """Read frontmatter + body for one path. Optionally include parsed chunks.
+
+ On-disk frontmatter is the source of truth — the file_store cache may
+ lag a write that hasn't been picked up by the watcher yet.
+ """
+ meta = file_store.get_file_meta(path)
+ result: dict = {"path": path, "exists": False}
+ if meta is not None:
+ edges = file_store.get_edges(path)
+ result.update({
+ "exists": True,
+ "metadata": meta.metadata,
+ "link": [e.model_dump(exclude_none=True) for e in edges],
+ })
+
+ file_path = Path(path)
+ if file_path.is_file():
+ raw = file_path.read_text(encoding="utf-8")
+ post = frontmatter.loads(raw)
+ result["exists"] = True
+ result["content"] = post.content
+ result["metadata"] = dict(post.metadata)
+
+ if include_chunks:
+ chunks = await file_store.get_chunks(path)
+ result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks]
+ return result
+
+
+def list_files(
+ file_store,
+ *,
+ path_prefix: str | None = None,
+ tags: list[str] | None = None,
+ metadata: dict | None = None,
+ limit: int = 100,
+) -> dict:
+ """List indexed files filtered by frontmatter exact-match, tags, and prefix.
+
+ Returns {items: [{path, metadata}], count}.
+ """
+ metadata_filter = metadata or {}
+ tag_filter = tags or []
+ items: list[dict] = []
+ for path, meta in file_store.nodes.items():
+ if path_prefix and not path.startswith(path_prefix):
+ continue
+ md = meta.metadata or {}
+ if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
+ continue
+ if tag_filter:
+ file_tags = set(md.get("tags", []) or [])
+ if not all(t in file_tags for t in tag_filter):
+ continue
+ items.append({"path": path, "metadata": md})
+ if len(items) >= limit:
+ break
+ return {"items": items, "count": len(items)}
+
+
+def _edge_to_dict(file_meta, edge) -> dict:
+ return {
+ "path": file_meta.path,
+ "metadata": file_meta.metadata,
+ "predicate": edge.predicate,
+ "anchor": edge.anchor,
+ "alias": edge.alias,
+ "embed": edge.embed,
+ "source": edge.source,
+ "confidence": edge.confidence,
+ }
+
+
+def links_of(file_store, path: str) -> dict:
+ """Files that `path` links TO (resolved). Each entry carries the typed-edge predicate."""
+ return {
+ "path": path,
+ "links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)],
+ }
+
+
+def backlinks_of(file_store, path: str) -> dict:
+ """Files that link TO `path`. Each entry carries the typed-edge predicate."""
+ return {
+ "path": path,
+ "backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)],
+ }
+
+
+def wikilink_lookup(file_store, wikilink: str) -> dict:
+ """Resolve a `[[target]]` wikilink with full ambiguity context.
+
+ Distinct from `file_store.resolve_wikilink(target)` (which returns
+ the single resolved path or None) — this surface returns the rich
+ payload callers need to disambiguate:
+
+ unique resolution → {wikilink, path, exists: True,
+ ambiguous: False, candidates: [path]}
+ ambiguous → {wikilink, path: None, exists: False,
+ ambiguous: True, candidates: [...]}
+ dangling → {wikilink, path: None, exists: False,
+ ambiguous: False, candidates: []}
+ """
+ # Path-form (`a/b` or `a/b.md`): file_store already returns
+ # exactly the one path that exists, or None.
+ if "/" in wikilink or wikilink.endswith(".md"):
+ hit = file_store.resolve_wikilink(wikilink)
+ return {
+ "wikilink": wikilink,
+ "path": hit,
+ "exists": hit is not None,
+ "ambiguous": False,
+ "candidates": [hit] if hit else [],
+ }
+
+ # Stem-form: candidates list reveals 0/1/N resolution.
+ candidates = file_store.wikilink_candidates(wikilink)
+ if len(candidates) == 1:
+ return {
+ "wikilink": wikilink,
+ "path": candidates[0],
+ "exists": True,
+ "ambiguous": False,
+ "candidates": candidates,
+ }
+ return {
+ "wikilink": wikilink,
+ "path": None,
+ "exists": False,
+ "ambiguous": len(candidates) > 1,
+ "candidates": candidates,
+ }
+
+
+async def count_tokens(
+ token_counter,
+ *,
+ path: str | None = None,
+ text: str | None = None,
+) -> dict:
+ """Estimate tokens for a file body (frontmatter excluded) or raw text.
+
+ Powers the Maintainer's split-trigger. Exactly one of `path` / `text`
+ must be provided.
+ """
+ if path:
+ target = Path(path)
+ if not target.is_file():
+ return {"path": str(target), "error": "file not found"}
+ raw = target.read_text(encoding="utf-8")
+ post = frontmatter.loads(raw)
+ body = post.content
+ tokens = await token_counter.count(messages=[], text=body)
+ return {
+ "source": "file",
+ "path": str(target.resolve()),
+ "tokens": tokens,
+ "body_chars": len(body),
+ }
+ if text:
+ tokens = await token_counter.count(messages=[], text=text)
+ return {
+ "source": "text",
+ "tokens": tokens,
+ "body_chars": len(text),
+ }
+ return {"error": "one of `path` or `text` is required"}
+
+
+def iter_files(file_store) -> Iterator[tuple[str, FileMetadata]]:
+ """Walk every indexed (path, FileMetadata). Used by Maintainer scans.
+
+ Equivalent to `file_store.nodes.items()`, exposed here so consumers
+ don't have to know about the underlying cache attribute name.
+ """
+ return iter(file_store.nodes.items())
+
+
+# ===========================================================================
+# Section 3 — Projection queries (Vector / FTS / File Graph)
+# ===========================================================================
+#
+# Per `structure.md` §"核心引擎", the Vector index, FTS5 index, and File
+# Graph are downstream **projections** of the MFS — they can be wholly
+# rebuilt from disk. These functions are the read entry to those
+# projections; `Retriever` composes them with policy (V+K weighting,
+# graph BFS, intent routing) for ranked retrieval.
+
+
+async def vector_search(
+ file_store,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
+) -> list[FileChunk]:
+ """Vector similarity over the chunk-level Vector projection."""
+ return await file_store.vector_search(query, limit, chunk_filter)
+
+
+async def keyword_search(
+ file_store,
+ query: str,
+ limit: int,
+ chunk_filter: ChunkFilter | None = None,
+) -> list[FileChunk]:
+ """FTS5 keyword search over the chunk-level keyword projection."""
+ return await file_store.keyword_search(query, limit, chunk_filter)
+
+
+def expand_neighbors(
+ file_store,
+ seeds: Iterable[str],
+ *,
+ depth: int = 1,
+ direction: str = "both",
+) -> dict[str, int]:
+ """BFS over the File Graph projection. Returns {path: hop_distance}."""
+ return file_store.expand_neighbors(seeds, depth=depth, direction=direction)
+
+
+def extract_anchors(file_store, text: str) -> list[str]:
+ """Pull anchor paths from `[[target]]` references inside `text`."""
+ return file_store.extract_anchor_paths(text)
+
+
+async def chunks_by_paths(file_store, paths: Iterable[str]) -> list[FileChunk]:
+ """Batch fetch chunks across many paths (used by graph-walk retrieval)."""
+ return await file_store.get_chunks_by_paths(paths)
+
+
+def make_chunk_filter(
+ file_store,
+ *,
+ paths: list[str] | None = None,
+ tags: list[str] | None = None,
+ exclude_paths: list[str] | None = None,
+) -> ChunkFilter | None:
+ """Build a chunk-filter against the file_store's path/tag indexes."""
+ return file_store.filter(paths=paths, tags=tags, exclude_paths=exclude_paths)
+
+
+def all_ambiguous_wikilinks(file_store) -> dict[str, list[str]]:
+ """Every stem that resolves to >1 path. Used by Maintainer.lint."""
+ return file_store.all_ambiguous_wikilinks()
+
+
+# ===========================================================================
+# Section 4 — Agent toolkit
+# ===========================================================================
+#
+# `MemoryIO` adapts the read/write helpers above as `agentscope.tool`
+# callables, with a vault-root containment check on writes and an audit
+# trail every consumer can inspect after the agent's run. Used by the
+# Ingestor's ReActAgent.
+
+
+def _text_response(payload: Any) -> ToolResponse:
+ text = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2)
+ return ToolResponse(content=[TextBlock(type="text", text=text)])
+
+
+class MemoryIO:
+ """Agent-facing tool surface over the Memory File System.
+
+ Args:
+ file_store: The vault's FileStore (provides graph + index).
+ vault_root: Containment boundary — write tools refuse paths
+ that escape it.
+
+ Attributes:
+ audit: Every successful or failed write is appended here. Read
+ this after the agent's run to reconstruct the mutation trail.
+ """
+
+ _TOOL_NAMES = (
+ "memory_get",
+ "memory_list",
+ "memory_resolve_wikilink",
+ "memory_backlinks",
+ "memory_links",
+ "memory_create",
+ "memory_update",
+ "memory_property_update",
+ "memory_rename",
+ "memory_delete",
+ "memory_archive",
+ )
+
+ def __init__(self, file_store, vault_root: str | Path):
+ self.file_store = file_store
+ self.vault_root = Path(vault_root).resolve()
+ self.audit: list[dict] = []
+
+ def register_all(self, toolkit: Toolkit | None = None) -> Toolkit:
+ """Register every memory_* method on `toolkit` (or a fresh one)."""
+ toolkit = toolkit or Toolkit()
+ for name in self._TOOL_NAMES:
+ toolkit.register_tool_function(
+ getattr(self, name), namesake_strategy="override",
+ )
+ return toolkit
+
+ # -- Internals --------------------------------------------------------
+
+ def _resolve(self, p: str) -> Path:
+ pp = Path(p)
+ if not pp.is_absolute():
+ pp = self.vault_root / pp
+ return pp.resolve()
+
+ def _under_vault(self, p: Path) -> bool:
+ try:
+ p.relative_to(self.vault_root)
+ return True
+ except (ValueError, OSError):
+ return False
+
+ def _record(self, op: str, ok: bool, **fields) -> dict:
+ entry = {"op": op, "ok": ok, **fields}
+ self.audit.append(entry)
+ return entry
+
+ # -- Read tools (delegate to module-level helpers) --------------------
+
+ async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse:
+ """Read a memory file (frontmatter + body, optional chunks).
+
+ Args:
+ path (str): Absolute path to the file.
+ include_chunks (bool): Include parsed chunk metadata.
+ """
+ target = self._resolve(path)
+ result = await read_file(self.file_store, str(target), include_chunks=include_chunks)
+ return _text_response(result)
+
+ async def memory_list(
+ self,
+ path_prefix: str | None = None,
+ tags: list[str] | None = None,
+ metadata: dict | None = None,
+ limit: int = 100,
+ ) -> ToolResponse:
+ """List indexed vault files filtered by prefix, tags, and frontmatter.
+
+ Args:
+ path_prefix (str | None): Restrict to paths starting with this prefix.
+ tags (list[str] | None): All tags must be present on a file.
+ metadata (dict | None): Exact-match filter on frontmatter keys.
+ limit (int): Cap on returned items.
+ """
+ return _text_response(list_files(
+ self.file_store,
+ path_prefix=path_prefix, tags=tags,
+ metadata=metadata, limit=limit,
+ ))
+
+ async def memory_resolve_wikilink(self, wikilink: str) -> ToolResponse:
+ """Resolve a `[[wikilink]]` to an absolute path.
+
+ Args:
+ wikilink (str): The wikilink target, e.g. `Topic` or `topics/Topic`.
+ """
+ return _text_response(wikilink_lookup(self.file_store, wikilink))
+
+ async def memory_backlinks(self, path: str) -> ToolResponse:
+ """List files linking TO a given path.
+
+ Args:
+ path (str): Absolute path to inspect.
+ """
+ target = self._resolve(path)
+ return _text_response(backlinks_of(self.file_store, str(target)))
+
+ async def memory_links(self, path: str) -> ToolResponse:
+ """List files a given path links to.
+
+ Args:
+ path (str): Absolute path to inspect.
+ """
+ target = self._resolve(path)
+ return _text_response(links_of(self.file_store, str(target)))
+
+ # -- Write tools (delegate to write_*; record audit) ------------------
+
+ async def memory_create(
+ self,
+ path: str,
+ metadata: dict | None = None,
+ content: str = "",
+ overwrite: bool = False,
+ force: bool = False,
+ ) -> ToolResponse:
+ """Create a new markdown file in the vault.
+
+ Args:
+ path (str): Target path (resolved against vault_root if relative).
+ metadata (dict | None): YAML frontmatter for the new file.
+ content (str): Body markdown.
+ overwrite (bool): Allow overwriting an existing file.
+ force (bool): Allow creates that introduce stem-form wikilink ambiguity.
+ """
+ target = self._resolve(path)
+ if not self._under_vault(target):
+ entry = self._record("create", False, path=str(target),
+ error=f"path is outside vault_root {self.vault_root}")
+ return _text_response(entry)
+ ok, payload = write_create(
+ self.file_store, target,
+ metadata=dict(metadata or {}), content=content,
+ overwrite=overwrite, force=force,
+ )
+ entry = self._record("create", ok, path=str(target), result=payload)
+ return _text_response(entry)
+
+ async def memory_update(
+ self,
+ path: str,
+ old_string: str,
+ new_string: str,
+ replace_all: bool = False,
+ ) -> ToolResponse:
+ """Edit a file body by exact-string substitution.
+
+ Use a unique snippet for `old_string`. To append, pass the file's
+ tail as `old_string` and `tail + new_content` as `new_string`.
+
+ Args:
+ path (str): Absolute path to the file.
+ old_string (str): Exact text to replace (must be unique unless replace_all).
+ new_string (str): Replacement text.
+ replace_all (bool): Replace every occurrence instead of just one.
+ """
+ target = self._resolve(path)
+ if not self._under_vault(target):
+ entry = self._record("update", False, path=str(target),
+ error=f"path is outside vault_root {self.vault_root}")
+ return _text_response(entry)
+ ok, payload = write_update(target, old_string, new_string, replace_all=replace_all)
+ entry = self._record("update", ok, path=str(target), result=payload)
+ return _text_response(entry)
+
+ async def memory_property_update(self, path: str, key: str, value: Any = None) -> ToolResponse:
+ """Update one YAML frontmatter key on a file (value=null deletes it).
+
+ Args:
+ path (str): Absolute path to the file.
+ key (str): Frontmatter key.
+ value: New value, or null to delete the key.
+ """
+ target = self._resolve(path)
+ if not self._under_vault(target):
+ entry = self._record("property_update", False, path=str(target),
+ error=f"path is outside vault_root {self.vault_root}")
+ return _text_response(entry)
+ ok, payload = write_property_update(target, key, value)
+ entry = self._record("property_update", ok, path=str(target), result=payload)
+ return _text_response(entry)
+
+ async def memory_rename(self, old_path: str, new_path: str) -> ToolResponse:
+ """Rename a file and rewrite cross-vault wikilinks.
+
+ Args:
+ old_path (str): Current absolute path.
+ new_path (str): Target absolute path.
+ """
+ old_p = self._resolve(old_path)
+ new_p = self._resolve(new_path)
+ if not self._under_vault(old_p) or not self._under_vault(new_p):
+ entry = self._record("rename", False, old_path=str(old_p), new_path=str(new_p),
+ error=f"path is outside vault_root {self.vault_root}")
+ return _text_response(entry)
+ ok, payload = write_rename(self.file_store, self.vault_root, old_p, new_p)
+ entry = self._record("rename", ok, old_path=str(old_p), new_path=str(new_p), result=payload)
+ return _text_response(entry)
+
+ async def memory_delete(self, path: str) -> ToolResponse:
+ """Delete a file from the vault.
+
+ Args:
+ path (str): Absolute path to the file.
+ """
+ target = self._resolve(path)
+ if not self._under_vault(target):
+ entry = self._record("delete", False, path=str(target),
+ error=f"path is outside vault_root {self.vault_root}")
+ return _text_response(entry)
+ ok, payload = write_delete(target)
+ entry = self._record("delete", ok, path=str(target), result=payload)
+ return _text_response(entry)
+
+ async def memory_archive(self, path: str, archive_dir: str = "Archive") -> ToolResponse:
+ """Flip `status: archived` and move file under `//`.
+
+ Args:
+ path (str): Absolute path to the file.
+ archive_dir (str): Subdirectory name under vault_root for archives.
+ """
+ target = self._resolve(path)
+ if not self._under_vault(target):
+ entry = self._record("archive", False, path=str(target),
+ error=f"path is outside vault_root {self.vault_root}")
+ return _text_response(entry)
+ ok, payload = write_archive(self.vault_root, target, archive_dir)
+ entry = self._record("archive", ok, path=str(target), result=payload)
+ return _text_response(entry)
diff --git a/reme2/memory/retriever.py b/reme2/memory/retriever.py
new file mode 100644
index 00000000..4641ea80
--- /dev/null
+++ b/reme2/memory/retriever.py
@@ -0,0 +1,344 @@
+"""Retriever — read service. The Memory subsystem's read-side facade.
+
+Per the architecture blueprint, retrieval policy lives here (not in the
+file_store): query understanding (intent routing) + multi-channel
+fusion (V + K + graph BFS) + ranking. The file_store exposes only
+single-channel index primitives; this module composes them with
+strategy.
+
+Two surfaces:
+
+ `search(query, ...)` — pure relevance retrieval (V + K hybrid).
+ `graph_search(query, ...)` — V + K + graph fusion (context expansion
+ through wikilinks).
+
+`BaseRetriever` inherits `BaseStep` so concrete retrievers get the
+standard Step lifecycle + `self.file_store` property + per-instance
+configuration via constructor kwargs. The default `execute()` dispatch
+just runs `graph_search` so the retriever can also be used directly as
+a step inside a job pipeline; the MCP step shells in
+`reme2.mcp.steps.memory_retriever` instead bypass `execute` and call
+`search` / `graph_search` directly so they own the result-serialization
+shape.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from abc import abstractmethod
+from collections import defaultdict
+
+from ..component import R
+from ..component.base_step import BaseStep
+from ..schema import ChunkFilter, FileChunk
+from . import memory_io
+
+
+class BaseRetriever(BaseStep):
+ """Pluggable retrieval strategy.
+
+ Inherits `BaseStep`, so:
+ - `component_type = ComponentEnum.STEP` (for registry lookup).
+ - `self.file_store` resolves the configured file_store via the
+ standard kwargs / app_context route — no manual `_start` lookup.
+ - kwargs (`vector_weight`, `graph_weight`, …) flow through the
+ standard Step kwargs handshake.
+
+ Concrete subclasses implement `search` and `graph_search`.
+ """
+
+ @abstractmethod
+ async def search(
+ self,
+ query: str,
+ *,
+ max_results: int = 5,
+ min_score: float = 0.0,
+ chunk_filter: ChunkFilter | None = None,
+ ) -> list[FileChunk]:
+ """Pure-relevance retrieval (V + K hybrid by convention)."""
+
+ @abstractmethod
+ async def graph_search(
+ self,
+ query: str = "",
+ *,
+ seeds: list[str] | None = None,
+ max_results: int = 5,
+ min_score: float = 0.0,
+ chunk_filter: ChunkFilter | None = None,
+ # Per-call overrides; None = use this retriever's own defaults.
+ vector_weight: float | None = None,
+ graph_weight: float | None = None,
+ graph_depth: int | None = None,
+ graph_decay: float | None = None,
+ graph_direction: str | None = None,
+ graph_mode: str | None = None,
+ graph_per_path_cap: int | None = None,
+ anchor_expand: bool | None = None,
+ ) -> tuple[list[FileChunk], dict[str, int]]:
+ """Three-way fusion search (V + K + graph BFS over wikilinks).
+
+ Returns:
+ (chunks, hops) — `hops[path]` is the BFS distance from the
+ seed set to that path (0 = seed, 1 = 1-hop neighbor, …).
+ """
+
+ async def execute(self):
+ """Default Step entry point — dispatches to `graph_search`.
+
+ The retriever is normally invoked via `search` / `graph_search`
+ directly by the MCP step shells, but this lets the retriever
+ also be wired as a step inside a job pipeline (e.g. for
+ debugging / scripts). Reads RuntimeContext for the standard
+ query / max_results / min_score / seeds + filter args.
+ """
+ assert self.context is not None
+ ctx = self.context
+ chunk_filter = memory_io.make_chunk_filter(
+ self.file_store,
+ paths=ctx.get("paths") or None,
+ tags=ctx.get("tags") or None,
+ exclude_paths=ctx.get("exclude_paths") or None,
+ )
+ results, _hops = await self.graph_search(
+ query=ctx.get("query", "").strip(),
+ seeds=list(ctx.get("seeds") or []),
+ max_results=int(ctx.get("max_results", 5)),
+ min_score=float(ctx.get("min_score", 0.0)),
+ chunk_filter=chunk_filter,
+ )
+ ctx.response.success = True
+ ctx.response.answer = [c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results]
+
+
+@R.register("hybrid")
+class HybridRetriever(BaseRetriever):
+ """V + K (+ optional graph BFS) fusion retriever.
+
+ Composes the file_store's single-channel primitives (`vector_search`,
+ `keyword_search`, `expand_neighbors`, `extract_anchor_paths`,
+ `get_chunks_by_paths`). Knobs (`vector_weight`, `graph_weight`,
+ `graph_depth`, `graph_decay`, `graph_direction`, `graph_mode`,
+ `graph_per_path_cap`, `anchor_expand`, `candidate_multiplier`) are
+ constructor defaults; `graph_search` accepts per-call overrides for
+ the graph fusion knobs (any None → fall back to constructor default).
+ """
+
+ def __init__(
+ self,
+ vector_weight: float = 0.7,
+ graph_weight: float = 0.3,
+ graph_depth: int = 1,
+ graph_decay: float = 0.5,
+ graph_direction: str = "both",
+ graph_mode: str = "additive",
+ graph_per_path_cap: int = 3,
+ candidate_multiplier: float = 3.0,
+ anchor_expand: bool = True,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ if graph_mode not in ("additive", "boost"):
+ raise ValueError(f"graph_mode must be 'additive' or 'boost', got {graph_mode!r}")
+ self.vector_weight = vector_weight
+ self.graph_weight = graph_weight
+ self.graph_depth = graph_depth
+ self.graph_decay = graph_decay
+ self.graph_direction = graph_direction
+ self.graph_mode = graph_mode
+ self.graph_per_path_cap = graph_per_path_cap
+ self.candidate_multiplier = candidate_multiplier
+ self.anchor_expand = anchor_expand
+
+ async def search(
+ self,
+ query: str,
+ *,
+ max_results: int = 5,
+ min_score: float = 0.0,
+ chunk_filter: ChunkFilter | None = None,
+ ) -> list[FileChunk]:
+ """V + K hybrid retrieval (no graph). Owns the fusion policy.
+
+ Pipeline:
+ 1. If both channels are enabled → run V + K in parallel,
+ merge by unique_key with weighted score.
+ 2. If only one is enabled → return that channel's results
+ directly (no fusion to do).
+ 3. If neither → empty.
+ """
+ fs = self.file_store
+ candidates = min(200, max(1, int(max_results * self.candidate_multiplier)))
+ text_weight = 1.0 - self.vector_weight
+
+ if fs.vector_enabled and fs.fts_enabled:
+ v_task = memory_io.vector_search(fs, query, candidates, chunk_filter)
+ k_task = memory_io.keyword_search(fs, query, candidates, chunk_filter)
+ v_results, k_results = await asyncio.gather(v_task, k_task)
+
+ if not k_results:
+ results = v_results[:max_results]
+ elif not v_results:
+ results = k_results[:max_results]
+ else:
+ results = self._merge_vk(
+ v_results, k_results, self.vector_weight, text_weight,
+ )[:max_results]
+ elif fs.vector_enabled:
+ results = await memory_io.vector_search(fs, query, max_results, chunk_filter)
+ elif fs.fts_enabled:
+ results = await memory_io.keyword_search(fs, query, max_results, chunk_filter)
+ else:
+ results = []
+
+ if min_score > 0:
+ results = [r for r in results if r.score >= min_score]
+ return results
+
+ @staticmethod
+ def _merge_vk(
+ vector: list[FileChunk],
+ keyword: list[FileChunk],
+ vector_weight: float,
+ text_weight: float,
+ ) -> list[FileChunk]:
+ """Weighted V+K merge by unique_key. Vector first (canonical),
+ keyword adds its weighted subscore on collision or seeds fresh."""
+ merged: dict[str, FileChunk] = {}
+ for r in vector:
+ r.scores["score"] = r.scores.get("vector", 0.0) * vector_weight
+ merged[r.unique_key] = r
+ for r in keyword:
+ key = r.unique_key
+ k = r.scores.get("keyword", 0.0)
+ if key in merged:
+ merged[key].scores["score"] += k * text_weight
+ else:
+ r.scores["score"] = k * text_weight
+ merged[key] = r
+ results = list(merged.values())
+ results.sort(key=lambda c: c.score, reverse=True)
+ return results
+
+ async def graph_search(
+ self,
+ query: str = "",
+ *,
+ seeds: list[str] | None = None,
+ max_results: int = 5,
+ min_score: float = 0.0,
+ chunk_filter: ChunkFilter | None = None,
+ vector_weight: float | None = None,
+ graph_weight: float | None = None,
+ graph_depth: int | None = None,
+ graph_decay: float | None = None,
+ graph_direction: str | None = None,
+ graph_mode: str | None = None,
+ graph_per_path_cap: int | None = None,
+ anchor_expand: bool | None = None,
+ ) -> tuple[list[FileChunk], dict[str, int]]:
+ # Resolve per-call overrides → constructor defaults.
+ vw = self.vector_weight if vector_weight is None else float(vector_weight)
+ gw = self.graph_weight if graph_weight is None else float(graph_weight)
+ gd = self.graph_depth if graph_depth is None else int(graph_depth)
+ gdc = self.graph_decay if graph_decay is None else float(graph_decay)
+ gdr = self.graph_direction if graph_direction is None else graph_direction
+ gm = self.graph_mode if graph_mode is None else graph_mode
+ gpc = self.graph_per_path_cap if graph_per_path_cap is None else int(graph_per_path_cap)
+ ae = self.anchor_expand if anchor_expand is None else bool(anchor_expand)
+
+ if gm not in ("additive", "boost"):
+ raise ValueError(f"graph_mode must be 'additive' or 'boost', got {gm!r}")
+ if not (0.0 <= vw <= 1.0):
+ raise ValueError(f"vector_weight must be in [0,1], got {vw}")
+ if not (0.0 <= gw <= 1.0):
+ raise ValueError(f"graph_weight must be in [0,1], got {gw}")
+
+ explicit_seeds = list(seeds or [])
+ if not query and not explicit_seeds:
+ raise ValueError("graph_search: query or seeds must be provided")
+
+ candidate_count = max(max_results, int(max_results * self.candidate_multiplier))
+ fs = self.file_store
+
+ # 1. V + K in parallel (each is a no-op when its backend is disabled).
+ if query:
+ v_task = memory_io.vector_search(fs, query, candidate_count, chunk_filter)
+ k_task = memory_io.keyword_search(fs, query, candidate_count, chunk_filter)
+ v_results, k_results = await asyncio.gather(v_task, k_task)
+ else:
+ v_results, k_results = [], []
+
+ # 2. Build seed set.
+ seed_paths: set[str] = set()
+ for c in v_results:
+ seed_paths.add(c.path)
+ for c in k_results:
+ seed_paths.add(c.path)
+ for p in explicit_seeds:
+ if p in fs:
+ seed_paths.add(p)
+ if ae and query:
+ seed_paths.update(memory_io.extract_anchors(fs, query))
+
+ # 3. Graph expansion.
+ if gw > 0 and seed_paths and gd >= 0:
+ hops = memory_io.expand_neighbors(fs, seed_paths, depth=gd, direction=gdr)
+ else:
+ hops = {}
+ graph_scores: dict[str, float] = {p: gdc ** h for p, h in hops.items()}
+
+ # 4. Pull graph-only chunks (paths in expansion but not in V∪K).
+ # Skip in 'boost' mode — boost only re-ranks V∪K, never adds candidates.
+ vk_paths = {c.path for c in v_results} | {c.path for c in k_results}
+ if gm == "additive" and hops:
+ extra_paths = set(hops) - vk_paths
+ if chunk_filter is not None and chunk_filter.resolved_paths is not None:
+ extra_paths &= chunk_filter.resolved_paths
+ extra_chunks = await memory_io.chunks_by_paths(fs, extra_paths)
+ # Per-path cap so a hub topic with N chunks doesn't flood results.
+ by_path: dict[str, list] = defaultdict(list)
+ for c in extra_chunks:
+ by_path[c.path].append(c)
+ graph_only_chunks: list[FileChunk] = []
+ for path_chunks in by_path.values():
+ path_chunks.sort(key=lambda c: c.start_line)
+ graph_only_chunks.extend(path_chunks[:gpc])
+ else:
+ graph_only_chunks = []
+
+ # 5. Merge by unique_key. Vector first (canonical), keyword fills its
+ # subscore on existing entries, graph-only chunks come in fresh.
+ pool: dict = {}
+ for c in v_results:
+ pool[c.unique_key] = c
+ for c in k_results:
+ existing = pool.get(c.unique_key)
+ if existing is None:
+ pool[c.unique_key] = c
+ else:
+ existing.scores["keyword"] = c.scores.get("keyword", 0.0)
+ for c in graph_only_chunks:
+ pool.setdefault(c.unique_key, c)
+
+ # 6. Final scoring.
+ for c in pool.values():
+ v = c.scores.get("vector", 0.0)
+ k = c.scores.get("keyword", 0.0)
+ g = graph_scores.get(c.path, 0.0)
+ vk = vw * v + (1.0 - vw) * k
+ if gm == "boost":
+ final = vk * (1.0 + gw * g)
+ else:
+ final = (1.0 - gw) * vk + gw * g
+ c.scores["graph"] = g
+ c.scores["score"] = final
+
+ # 7. Rank + filter + slice.
+ results = sorted(pool.values(), key=lambda c: c.score, reverse=True)
+ if min_score > 0:
+ results = [r for r in results if r.score >= min_score]
+ results = results[:max_results]
+
+ return results, hops
diff --git a/reme2/file_based/summarizer.py b/reme2/memory/summarizer.py
similarity index 95%
rename from reme2/file_based/summarizer.py
rename to reme2/memory/summarizer.py
index c5ba5f21..62335c9b 100644
--- a/reme2/file_based/summarizer.py
+++ b/reme2/memory/summarizer.py
@@ -18,16 +18,16 @@ class Summarizer(BaseStep):
"""Summarizer step for summarizing memory messages."""
def __init__(
- self,
- working_dir: str,
- memory_dir: str,
- memory_compact_threshold: int,
- toolkit: Toolkit | None = None,
- console_enabled: bool = False,
- timezone: str | None = None,
- add_thinking_block: bool = True,
- as_token_counter: HuggingFaceTokenCounter | None = None,
- **kwargs,
+ self,
+ working_dir: str,
+ memory_dir: str,
+ memory_compact_threshold: int,
+ toolkit: Toolkit | None = None,
+ console_enabled: bool = False,
+ timezone: str | None = None,
+ add_thinking_block: bool = True,
+ as_token_counter: HuggingFaceTokenCounter | None = None,
+ **kwargs,
):
"""Initialize the summarizer step.
@@ -227,10 +227,10 @@ class Summarizer(BaseStep):
return total
async def _format_msgs_to_str(
- self,
- messages: list[Msg],
- memory_compact_threshold: int,
- include_thinking: bool = True,
+ self,
+ messages: list[Msg],
+ memory_compact_threshold: int,
+ include_thinking: bool = True,
) -> str:
"""Format list of messages to a single formatted string.
diff --git a/reme2/file_based/summarizer.yaml b/reme2/memory/summarizer.yaml
similarity index 99%
rename from reme2/file_based/summarizer.yaml
rename to reme2/memory/summarizer.yaml
index a9cb59e6..7b448df6 100644
--- a/reme2/file_based/summarizer.yaml
+++ b/reme2/memory/summarizer.yaml
@@ -1,14 +1,14 @@
user_message: |
Memory Pre-compression Flush Cycle.
-
+
The current session is about to enter the automatic compression phase. Please capture persistent memory AND session reflections, then write them to disk.
-
+
Current date: {date}
Working directory: {working_dir}
-
+
# Task
Immediately store persistent memory and reflections to: {memory_dir}/YYYY-MM-DD.md
-
+
# Workflow
1. Extract and synthesize content from the current session:
- Persistent Memory: Facts, user profile updates, project states, and important events.
@@ -17,7 +17,7 @@ user_message: |
- If the file doesn’t exist, use `write` tool directly.
- If the file exists, intelligently merge new information with existing content, prefer using `edit` to update specific sections.
- Use `write` to overwrite the entire file only if substantial restructuring is required.
-
+
# Principles
- Intelligently merge new information with existing content:
- Categorize clearly (e.g., separate "Factual Memory" from "Reflections & Logic").
@@ -32,15 +32,15 @@ user_message: |
user_message_zh: |
预压缩内存刷新轮次。
-
+
当前会话即将进入自动压缩阶段；请将持久化记忆与经验反思捕获并写入磁盘。
-
+
当前日期：{date}
工作目录：{working_dir}
-
+
# 任务
立即存储持久化记忆与反思（使用路径 {memory_dir}/YYYY-MM-DD.md）。
-
+
# 工作流程
1. 从当前会话中提取并综合两类内容：
- 持久化记忆：客观事实、用户信息更新、项目状态及重要事件。
@@ -48,7 +48,7 @@ user_message_zh: |
2. `read` {memory_dir}/YYYY-MM-DD.md（如文件不存在，会返回错误提示）
- 若文件不存在，直接使用 `write` 工具写入。
- 若文件已存在，智能合并新信息与现有内容，尽可能使用 `edit` 更新特定部分，仅在需要大幅重构时使用 `write` 覆盖整个文件。
-
+
# 原则
- 智能合并新信息与现有内容：
- 将内容进行清晰的分类（例如明确区分“事实记忆”与“反思与逻辑”）。
diff --git a/reme2/reme_backup.py b/reme2/reme_backup.py
index 99fd369f..16ba6a4e 100644
--- a/reme2/reme_backup.py
+++ b/reme2/reme_backup.py
@@ -13,7 +13,7 @@ from .application import Application
from .component import R, RuntimeContext
from .config import parse_args
from .enumeration import ComponentEnum
-from .file_based.summarizer import Summarizer
+from .memory.summarizer import Summarizer
from .utils import run_coro_safely
@@ -21,17 +21,17 @@ class ReMe(Application):
"""ReMe memory management application."""
async def summarize(
- self,
- messages: list[Msg],
- as_llm: str | ChatModelBase = "default",
- as_llm_formatter: str | FormatterBase = "default",
- as_token_counter: str | TokenCounterBase | HuggingFaceTokenCounter = "default",
- toolkit: Toolkit | None = None,
- language: str = "zh",
- max_input_length: float = 128 * 1024,
- compact_ratio: float = 0.7,
- timezone: str | None = None,
- add_thinking_block: bool = True,
+ self,
+ messages: list[Msg],
+ as_llm: str | ChatModelBase = "default",
+ as_llm_formatter: str | FormatterBase = "default",
+ as_token_counter: str | TokenCounterBase | HuggingFaceTokenCounter = "default",
+ toolkit: Toolkit | None = None,
+ language: str = "zh",
+ max_input_length: float = 128 * 1024,
+ compact_ratio: float = 0.7,
+ timezone: str | None = None,
+ add_thinking_block: bool = True,
) -> str:
"""Summarize and compact memory messages.
@@ -84,7 +84,7 @@ class ReMe(Application):
async def memory_search(self, query: str, max_results: int = 5, min_score: float = 0.1) -> str:
"""Search memory for relevant entries."""
- from .file_based.memory_search import MemorySearch
+ from .memory.memory_retriever import MemorySearch
try:
search_step = MemorySearch()
@@ -94,25 +94,25 @@ class ReMe(Application):
return str(e)
async def dream(
- self,
- as_llm: str | ChatModelBase = "default",
- as_llm_formatter: str | FormatterBase = "default",
- as_token_counter: str | TokenCounterBase = "default",
- toolkit: Toolkit | None = None,
- language: str = "zh",
- timezone: str | None = None,
+ self,
+ as_llm: str | ChatModelBase = "default",
+ as_llm_formatter: str | FormatterBase = "default",
+ as_token_counter: str | TokenCounterBase = "default",
+ toolkit: Toolkit | None = None,
+ language: str = "zh",
+ timezone: str | None = None,
) -> str:
"""Process and consolidate memories in background."""
return ""
async def proactive(
- self,
- as_llm: str | ChatModelBase = "default",
- as_llm_formatter: str | FormatterBase = "default",
- as_token_counter: str | TokenCounterBase = "default",
- toolkit: Toolkit | None = None,
- language: str = "zh",
- timezone: str | None = None,
+ self,
+ as_llm: str | ChatModelBase = "default",
+ as_llm_formatter: str | FormatterBase = "default",
+ as_token_counter: str | TokenCounterBase = "default",
+ toolkit: Toolkit | None = None,
+ language: str = "zh",
+ timezone: str | None = None,
) -> str:
"""Generate proactive memory insights."""
return ""
diff --git a/reme2/schema/__init__.py b/reme2/schema/__init__.py
index b0c9310f..37751258 100644
--- a/reme2/schema/__init__.py
+++ b/reme2/schema/__init__.py
@@ -5,7 +5,9 @@ from .as_msg_stat import AsBlockStat, AsMsgStat
from .base_node import BaseNode
from .chunk_filter import ChunkFilter
from .file_chunk import FileChunk
+from .file_edge import FileEdge
from .file_metadata import FileMetadata
+from .parsed_file import ParsedFile
from .request import Request
from .response import Response
from .stream_chunk import StreamChunk
@@ -19,7 +21,9 @@ __all__ = [
"BaseNode",
"ChunkFilter",
"FileChunk",
+ "FileEdge",
"FileMetadata",
+ "ParsedFile",
"Request",
"Response",
"StreamChunk",
diff --git a/reme2/schema/chunk_filter.py b/reme2/schema/chunk_filter.py
index 9cd4870f..b2b87b93 100644
--- a/reme2/schema/chunk_filter.py
+++ b/reme2/schema/chunk_filter.py
@@ -1,23 +1,23 @@
-"""Filter for chunk-store search.
+"""Filter for chunk search.
User-facing fields (paths/tags/exclude_paths) describe metadata-level intent.
-FileGraph compiles them into `resolved_paths` (the concrete path set that
-ChunkStore actually consumes for filtering).
+file_store compiles them into `resolved_paths` (the concrete path set that
+the chunk search actually consumes for filtering).
"""
from pydantic import BaseModel, Field
class ChunkFilter(BaseModel):
- """User-facing search filter, resolved by FileGraph into a path set.
+ """User-facing search filter, resolved by file_store into a path set.
User input fields:
paths: include only paths starting with any of these prefixes
tags: include only files whose metadata contains ALL these tags
exclude_paths: exclude paths starting with any of these prefixes
- Compiled field (set by FileGraph.filter):
- resolved_paths: concrete path set to restrict ChunkStore search.
+ Compiled field (set by file_store.filter):
+ resolved_paths: concrete path set to restrict chunk search.
None = no restriction (all chunks).
Empty set = no chunks match (search returns empty).
"""
@@ -43,7 +43,7 @@ class ChunkFilter(BaseModel):
return True
def match_path(self, path: str) -> bool:
- """Match a path against the resolved path set (used by ChunkStore)."""
+ """Match a path against the resolved path set (used by FileStore)."""
if self.resolved_paths is None:
return True
return path in self.resolved_paths
diff --git a/reme2/schema/file_edge.py b/reme2/schema/file_edge.py
new file mode 100644
index 00000000..b1b68deb
--- /dev/null
+++ b/reme2/schema/file_edge.py
@@ -0,0 +1,43 @@
+"""FileEdge — typed wikilink edge between vault files.
+
+Replaces the prior `link: list[str]` model on FileMetadata. An edge
+captures both the bare wikilink shape (`target` + optional anchor /
+alias / embed prefix) AND the typed-edge predicate that turns a plain
+`[[X]]` reference into a graph relation.
+
+Sources:
+ "regex" — extracted from inline syntax (`[[X]]`, `[pred:: [[X]]]`)
+ "frontmatter" — inferred from a frontmatter key acting as predicate
+ (e.g. `author: "[[John]]"` → predicate="author")
+ "llm" — produced by an IE pipeline; should set `confidence`
+
+The `target` field stays raw (e.g. `"X"` or `"topics/X"`) — resolution
+to an absolute path is done by the file_store via `resolve_wikilink`.
+"""
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class FileEdge(BaseModel):
+ target: str = Field(..., description="Raw wikilink target as written in source.")
+ predicate: str | None = Field(
+ default=None,
+ description="Typed-edge predicate (Dataview-style). None for plain wikilinks.",
+ )
+ anchor: str | None = Field(default=None, description="Heading or block anchor (after #).")
+ alias: str | None = Field(default=None, description="Display alias (after |).")
+ embed: bool = Field(default=False, description="True for `![[X]]` embed prefix.")
+ source: Literal["regex", "frontmatter", "llm"] = Field(
+ default="regex",
+ description="Provenance of this edge.",
+ )
+ confidence: float | None = Field(
+ default=None,
+ description="LLM confidence (0..1). None for regex/frontmatter.",
+ )
+
+ @property
+ def is_typed(self) -> bool:
+ return self.predicate is not None
diff --git a/reme2/schema/file_graph.py b/reme2/schema/file_graph.py
deleted file mode 100644
index 0dc7a65c..00000000
--- a/reme2/schema/file_graph.py
+++ /dev/null
@@ -1,119 +0,0 @@
-import json
-from collections import defaultdict
-from pathlib import Path
-
-from .chunk_filter import ChunkFilter
-from .file_metadata import FileMetadata
-
-
-class FileGraph:
-
- def __init__(self) -> None:
- self._nodes: dict[str, FileMetadata] = {}
- self._backlinks: dict[str, set[str]] = defaultdict(set)
-
- # -- CRUD ----------------------------------------------------------------
-
- def create(self, *metadatas: FileMetadata) -> None:
- for metadata in metadatas:
- path = metadata.path
- if path in self._nodes:
- self._remove_forward(self._nodes[path])
- self._nodes[path] = metadata
- for metadata in metadatas:
- self._add_forward(metadata)
-
- def read(self, path: str) -> FileMetadata | None:
- return self._nodes.get(path)
-
- def update(self, path: str, **fields) -> FileMetadata | None:
- metadata = self._nodes.get(path)
- if metadata is None:
- return None
- self._remove_forward(metadata)
- updated = metadata.model_copy(update=fields)
- self._nodes[path] = updated
- self._add_forward(updated)
- return updated
-
- def delete(self, path: str) -> FileMetadata | None:
- metadata = self._nodes.pop(path, None)
- if metadata is None:
- return None
- self._remove_forward(metadata)
- self._backlinks.pop(path, None)
- return metadata
-
- # -- Link queries --------------------------------------------------------
-
- def get_links(self, path: str) -> list[FileMetadata]:
- metadata = self._nodes.get(path)
- if metadata is None:
- return []
- return [self._nodes[link] for link in metadata.link if link in self._nodes]
-
- def get_backlinks(self, path: str) -> list[FileMetadata]:
- return [
- self._nodes[src]
- for src in self._backlinks.get(path, set())
- if src in self._nodes
- ]
-
- def filter(
- self,
- paths: list[str] | None = None,
- tags: list[str] | None = None,
- exclude_paths: list[str] | None = None,
- ) -> ChunkFilter:
- cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
- if cf.is_empty():
- return cf
- cf.resolved_paths = {
- path for path, meta in self._nodes.items()
- if cf.match_metadata(path, meta.metadata)
- }
- return cf
-
- # -- Index helpers -------------------------------------------------------
-
- def _add_forward(self, metadata: FileMetadata) -> None:
- for link in metadata.link:
- self._backlinks[link].add(metadata.path)
-
- def _remove_forward(self, metadata: FileMetadata) -> None:
- for link in metadata.link:
- self._backlinks[link].discard(metadata.path)
-
- # -- Persistence ---------------------------------------------------------
-
- def save(self, path: str | Path) -> None:
- path = Path(path)
- path.parent.mkdir(parents=True, exist_ok=True)
- raw = {p: meta.model_dump(mode="json") for p, meta in self._nodes.items()}
- content = json.dumps(raw, ensure_ascii=False)
- temp = path.with_suffix(".tmp")
- temp.write_text(content, encoding="utf-8")
- temp.replace(path)
-
- @classmethod
- def load(cls, path: str | Path) -> "FileGraph":
- path = Path(path)
- graph = cls()
- if not path.exists():
- return graph
- raw: dict = json.loads(path.read_text(encoding="utf-8"))
- nodes = [FileMetadata(**meta) for meta in raw.values()]
- graph.create(*nodes)
- return graph
-
- # -- Dunder --------------------------------------------------------------
-
- @property
- def nodes(self) -> dict[str, FileMetadata]:
- return self._nodes
-
- def __len__(self) -> int:
- return len(self._nodes)
-
- def __contains__(self, path: str) -> bool:
- return path in self._nodes
diff --git a/reme2/schema/file_metadata.py b/reme2/schema/file_metadata.py
index 86caf06d..828eee58 100644
--- a/reme2/schema/file_metadata.py
+++ b/reme2/schema/file_metadata.py
@@ -5,5 +5,4 @@ class FileMetadata(BaseModel):
file: str = Field(...)
path: str = Field(...)
st_mtime: float = Field(...)
- link: list[str] = Field(default_factory=list)
metadata: dict = Field(default_factory=dict)
diff --git a/reme2/schema/parsed_file.py b/reme2/schema/parsed_file.py
new file mode 100644
index 00000000..8e583cec
--- /dev/null
+++ b/reme2/schema/parsed_file.py
@@ -0,0 +1,28 @@
+"""ParsedFile — Parser's single output, carrying everything one file pass produces.
+
+Inherits `FileMetadata` (file / path / st_mtime / metadata) and adds the
+parsed contents:
+
+ chunks: list[FileChunk] — semantic blocks (text + position + hash);
+ embeddings ARE attached here — the parser
+ owns the embedding step (with hash-diff
+ cache via `existing_chunks` parameter).
+ edges: list[FileEdge] — wikilink graph edges (regex / frontmatter /
+ LLM provenance preserved per-edge).
+
+The Watcher passes a `ParsedFile` into `file_store.upsert_parsed(...)`,
+which dispatches the three sub-payloads (meta / edges / chunks) to the
+appropriate persistence pipelines while keeping the upsert atomic at
+the store layer.
+"""
+
+from pydantic import Field
+
+from .file_chunk import FileChunk
+from .file_edge import FileEdge
+from .file_metadata import FileMetadata
+
+
+class ParsedFile(FileMetadata):
+ chunks: list[FileChunk] = Field(default_factory=list)
+ edges: list[FileEdge] = Field(default_factory=list)
diff --git a/reme2/schema/vault/__init__.py b/reme2/schema/vault/__init__.py
new file mode 100644
index 00000000..1bf89733
--- /dev/null
+++ b/reme2/schema/vault/__init__.py
@@ -0,0 +1,41 @@
+"""Vault business schemas — domain models for the markdown vault.
+
+These are the *business objects* the user stores in the vault (Topic,
+Event), distinct from the engine schemas in `reme2/schema/` (FileMetadata,
+FileChunk, ChunkFilter — internal data types).
+
+Single Topic class covers all categories under topics/ (no satellite /
+tentacle subclassing); folder-topic is identified by path convention.
+
+Lives under `reme2/schema/vault/` (not in `reme2/mcp/`) so memory
+services (Maintainer.lint, Ingestor) can validate frontmatter without
+importing the MCP transport layer — that's what was creating the
+mcp ↔ memory dependency cycle.
+"""
+
+from .event import Event, EventStatus
+from .frontmatter import VaultBaseFrontmatter, parse_frontmatter
+from .registry import ALL_KNOWN_CATEGORIES, schema_for
+from .topic import (
+ INDEX_CATEGORIES,
+ JUDGMENT_CATEGORIES,
+ CONTENT_CATEGORIES,
+ Confidence,
+ Topic,
+ TopicCategory,
+)
+
+__all__ = [
+ "ALL_KNOWN_CATEGORIES",
+ "CONTENT_CATEGORIES",
+ "Confidence",
+ "Event",
+ "EventStatus",
+ "INDEX_CATEGORIES",
+ "JUDGMENT_CATEGORIES",
+ "Topic",
+ "TopicCategory",
+ "VaultBaseFrontmatter",
+ "parse_frontmatter",
+ "schema_for",
+]
diff --git a/reme2/schema/vault/event.py b/reme2/schema/vault/event.py
new file mode 100644
index 00000000..0ec580f7
--- /dev/null
+++ b/reme2/schema/vault/event.py
@@ -0,0 +1,16 @@
+"""Event schema — task process records under events/{date}/{name}/."""
+
+from typing import Literal
+
+from .frontmatter import VaultBaseFrontmatter
+
+EventStatus = Literal["active", "distilled", "archived"]
+
+
+class Event(VaultBaseFrontmatter):
+ """events/{YYYY-MM-DD}/{name}/{name}.md."""
+
+ category: Literal["event"] = "event" # type: ignore[assignment]
+ status: EventStatus = "active"
+ topics: list[str] = []
+ originSessionId: str | None = None
diff --git a/reme2/schema/vault/frontmatter.py b/reme2/schema/vault/frontmatter.py
new file mode 100644
index 00000000..4b5a3c62
--- /dev/null
+++ b/reme2/schema/vault/frontmatter.py
@@ -0,0 +1,38 @@
+"""Common frontmatter schema shared by all vault files."""
+
+from datetime import date, datetime
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class VaultBaseFrontmatter(BaseModel):
+ """Fields present on every vault file."""
+
+ model_config = ConfigDict(extra="allow")
+
+ title: str = Field(...)
+ description: str = Field(default="")
+ tags: list[str] = Field(default_factory=list)
+ category: str = Field(...)
+ created: date | None = None
+ updated: date | None = None
+
+ @field_validator("created", "updated", mode="before")
+ @classmethod
+ def _coerce_date(cls, v):
+ if v is None or isinstance(v, date):
+ return v
+ if isinstance(v, datetime):
+ return v.date()
+ if isinstance(v, str):
+ return datetime.fromisoformat(v).date()
+ return v
+
+
+def parse_frontmatter(raw: dict[str, Any]) -> VaultBaseFrontmatter:
+ """Tolerant parse — never raises; missing required fields fall back to defaults."""
+ safe = dict(raw or {})
+ safe.setdefault("title", safe.get("name") or "")
+ safe.setdefault("category", "unknown")
+ return VaultBaseFrontmatter.model_validate(safe)
diff --git a/reme2/schema/vault/registry.py b/reme2/schema/vault/registry.py
new file mode 100644
index 00000000..8d539087
--- /dev/null
+++ b/reme2/schema/vault/registry.py
@@ -0,0 +1,27 @@
+"""Map vault category → strict pydantic model for validation.
+
+Used by the Maintainer to derive lint rules from schema instead of
+hardcoding. Two business object schemas:
+ - event → Event
+ - everything else → Topic (judgment subset enforced via model_validator)
+"""
+
+from typing import get_args
+
+from pydantic import BaseModel
+
+from .event import Event
+from .topic import Topic, TopicCategory
+
+_REGISTRY: dict[str, type[BaseModel]] = {"event": Event}
+for _cat in get_args(TopicCategory):
+ _REGISTRY[_cat] = Topic
+
+ALL_KNOWN_CATEGORIES: frozenset[str] = frozenset(_REGISTRY.keys())
+
+
+def schema_for(category: str | None) -> type[BaseModel] | None:
+ """Return the pydantic schema bound to a category, or None for unknown."""
+ if not category:
+ return None
+ return _REGISTRY.get(category)
diff --git a/reme2/schema/vault/topic.py b/reme2/schema/vault/topic.py
new file mode 100644
index 00000000..c54f6d49
--- /dev/null
+++ b/reme2/schema/vault/topic.py
@@ -0,0 +1,47 @@
+"""Topic schema — every .md under topics/ is a Topic.
+
+No satellite/tentacle/folder note subclassing. Folder topic = Topic whose
+filename equals its parent directory name (path convention; not a schema field).
+"""
+
+from typing import Literal
+
+from pydantic import model_validator
+
+from .frontmatter import VaultBaseFrontmatter
+
+# Topic categories grouped by usage:
+# - Index categories: usually appear as folder topics; may have ticker/market
+# - Content categories: cluster siblings; thesis/model/questions need confidence
+INDEX_CATEGORIES = {"company", "sector", "concept", "method", "tool", "profile"}
+JUDGMENT_CATEGORIES = {"thesis", "model", "questions"}
+CONTENT_CATEGORIES = JUDGMENT_CATEGORIES | {"fundamentals"}
+
+TopicCategory = Literal[
+ "company", "sector", "concept", "method", "tool", "profile",
+ "thesis", "model", "questions", "fundamentals",
+]
+Confidence = Literal["⏳", "✅", "❌"]
+
+
+class Topic(VaultBaseFrontmatter):
+ """topics/{folder}/{name}.md — long-lived cognitive memory node.
+
+ Folder topic is identified by path convention: filename stem == parent
+ folder name. Not represented in this schema directly; checked at runtime
+ via `Path(p).stem == Path(p).parent.name`.
+ """
+
+ category: TopicCategory # type: ignore[assignment]
+ market: str | None = None
+ ticker: str | None = None
+ confidence: Confidence | None = None
+
+ @model_validator(mode="after")
+ def _confidence_required_for_judgments(self):
+ if self.category in JUDGMENT_CATEGORIES and self.confidence is None:
+ raise ValueError(
+ f"category={self.category} requires explicit confidence "
+ f"(one of ⏳ / ✅ / ❌)"
+ )
+ return self
diff --git a/reme2/utils/__init__.py b/reme2/utils/__init__.py
index 19595659..d089c295 100644
--- a/reme2/utils/__init__.py
+++ b/reme2/utils/__init__.py
@@ -1,17 +1,24 @@
"""Utility modules"""
from .case_converter import camel_to_snake, snake_to_camel
-from .chunking_utils import chunk_markdown
from .common_utils import hash_text, execute_stream_task, run_coro_safely
from .logger_utils import get_logger
from .logo_utils import print_logo
from .similarity_utils import cosine_similarity, batch_cosine_similarity
from .singleton import singleton
+from .wikilink import (
+ InlineField,
+ extract_inline_fields,
+ extract_typed_edges,
+ extract_wikilinks,
+ extract_wikilinks_from_metadata,
+ parse_wikilinks,
+ parse_wikilinks_from_metadata,
+)
__all__ = [
"camel_to_snake",
"snake_to_camel",
- "chunk_markdown",
"hash_text",
"execute_stream_task",
"run_coro_safely",
@@ -20,4 +27,11 @@ __all__ = [
"cosine_similarity",
"batch_cosine_similarity",
"singleton",
+ "InlineField",
+ "extract_wikilinks",
+ "extract_wikilinks_from_metadata",
+ "parse_wikilinks",
+ "parse_wikilinks_from_metadata",
+ "extract_inline_fields",
+ "extract_typed_edges",
]
diff --git a/reme2/utils/chunking_utils.py b/reme2/utils/chunking_utils.py
deleted file mode 100644
index 9e94d5d3..00000000
--- a/reme2/utils/chunking_utils.py
+++ /dev/null
@@ -1,144 +0,0 @@
-"""Markdown file chunking utilities.
-
-Provides functionality to split Markdown documents into smaller chunks
-while maintaining overlap between consecutive chunks for context preservation.
-"""
-
-from .common_utils import hash_text
-from ..schema import FileChunk
-
-
-def chunk_markdown(
- text: str,
- path: str,
- chunk_tokens: int,
- overlap: int,
-) -> list[FileChunk]:
- """Split Markdown text into chunks with configurable size and overlap.
-
- Implements a sliding window approach to chunk Markdown content while
- preserving context through overlap between consecutive chunks. Token
- counts are approximated using a 1:4 ratio (1 token ≈ 4 characters).
-
- Args:
- text: Input Markdown text to be chunked.
- path: File path identifier for the source document.
- chunk_tokens: Maximum number of tokens per chunk. Will be converted
- to characters using the 1:4 ratio, with a minimum of 32 characters.
- overlap: Number of overlapping tokens between consecutive chunks.
- Helps maintain context across chunk boundaries.
-
- Returns:
- A list of FileChunk objects, each containing:
- - id: Unique identifier based on path, line numbers, and hash
- - path: The source file path
- - start_line: Starting line number (1-indexed)
- - end_line: Ending line number (1-indexed)
- - text: The chunk content
- - hash: SHA-256 hash of the chunk content
-
- Examples:
- >>> text = "# Header\\nParagraph content here.\\n\\n## Subheader"
- >>> chunks = chunk_markdown(text, "doc.md", chunk_tokens=100, overlap=20)
- >>> len(chunks)
- 1
- >>> chunks[0].path
- 'doc.md'
- """
- if not text.strip():
- return []
-
- lines = text.split("\n")
-
- # Convert tokens to characters (~1 token = 4 chars)
- max_chars = max(32, chunk_tokens * 4)
- overlap_chars = max(0, overlap * 4)
-
- chunks: list[FileChunk] = []
-
- # Currently building chunk
- current: list[dict] = [] # [{'line': str, 'line_no': int}]
- current_chars = 0
-
- def flush() -> None:
- """Add current chunk to results list."""
- if not current:
- return
-
- first_entry = current[0]
- last_entry = current[-1]
-
- if not first_entry or not last_entry:
- return
-
- chunk_text = "\n".join([entry["line"] for entry in current])
- start_line = first_entry["line_no"]
- end_line = last_entry["line_no"]
-
- chunk_hash = hash_text(chunk_text)
-
- chunks.append(
- FileChunk(
- id=hash_text(f"{path}:{start_line}:{end_line}:{chunk_hash}:{len(chunks)}"),
- path=path,
- start_line=start_line,
- end_line=end_line,
- text=chunk_text,
- hash=chunk_hash,
- ),
- )
-
- def carry_overlap() -> None:
- """Keep overlapping part and clear the rest."""
- nonlocal current, current_chars
-
- if overlap_chars <= 0 or not current:
- current = []
- current_chars = 0
- return
-
- acc = 0
- kept = []
-
- # Collect lines from the end until reaching overlap size
- for j in range(len(current) - 1, -1, -1):
- entry = current[j]
- if not entry:
- continue
-
- acc += len(entry["line"]) + 1 # +1 for newline
- kept.insert(0, entry) # Insert at the beginning to maintain order
-
- if acc >= overlap_chars:
- break
-
- current = kept
- current_chars = sum(len(entry["line"]) + 1 for entry in kept)
-
- for i, line in enumerate(lines):
- line_no = i + 1
-
- # Split long lines into multiple segments
- segments = []
- if not line: # Empty line
- segments.append("")
- else:
- # If line is too long, split by maximum character count
- for start in range(0, len(line), max_chars):
- segments.append(line[start: start + max_chars])
-
- for segment in segments:
- line_size = len(segment) + 1 # +1 for newline
-
- # If adding current segment would exceed the limit, flush current chunk
- if current_chars + line_size > max_chars and current:
- flush()
- carry_overlap()
-
- current.append({"line": segment, "line_no": line_no})
- current_chars += line_size
-
- # Process the final chunk
- flush()
-
- return [c for c in chunks if c.text.strip()]
diff --git a/reme2/utils/common_utils.py b/reme2/utils/common_utils.py
index 513cb8ec..6c86d487 100644
--- a/reme2/utils/common_utils.py
+++ b/reme2/utils/common_utils.py
@@ -47,10 +47,10 @@ def hash_text(text: str, encoding: str = "utf-8") -> str:
async def execute_stream_task(
- stream_queue: asyncio.Queue,
- task: asyncio.Task,
- task_name: str | None = None,
- output_format: Literal["str", "bytes", "chunk"] = "str",
+ stream_queue: asyncio.Queue,
+ task: asyncio.Task,
+ task_name: str | None = None,
+ output_format: Literal["str", "bytes", "chunk"] = "str",
) -> AsyncGenerator[str | bytes | StreamChunk, None]:
"""Core stream flow execution logic.
diff --git a/reme2/utils/logger_utils.py b/reme2/utils/logger_utils.py
index b9648152..0c2eeb70 100644
--- a/reme2/utils/logger_utils.py
+++ b/reme2/utils/logger_utils.py
@@ -14,11 +14,11 @@ _initialized = False
def get_logger(
- log_dir: str = "logs",
- level: str = "INFO",
- log_to_console: bool = True,
- log_to_file: bool = True,
- force_init: bool = False,
+ log_dir: str = "logs",
+ level: str = "INFO",
+ log_to_console: bool = True,
+ log_to_file: bool = True,
+ force_init: bool = False,
):
"""Get a configured logger instance.
diff --git a/reme2/utils/vault_paths.py b/reme2/utils/vault_paths.py
new file mode 100644
index 00000000..52394bc7
--- /dev/null
+++ b/reme2/utils/vault_paths.py
@@ -0,0 +1,78 @@
+"""Vault path generation + naming disambiguation.
+
+Pure-Python helpers consumed by the MCP step shells (`sync`,
+`topic_create`) and any future flow that needs to materialize a path
+under the vault layout. No async, no MCP awareness — easy to unit-test.
+
+Wikilink uniqueness is a *graph* property, not a path-builder concern —
+see `reme2.component.file_store.BaseFileStore.collisions_after_create`
+and `BaseFileStore.all_ambiguous_wikilinks`.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from datetime import date as date_type
+from pathlib import Path
+
+
+def event_path(
+ vault_root: str | Path,
+ name: str,
+ on_date: date_type | str | None = None,
+ events_dir: str = "events",
+) -> Path:
+ """events/{YYYY-MM-DD}/{name}/{name}.md under the given vault root."""
+ if on_date is None:
+ on_date = date_type.today()
+ if isinstance(on_date, date_type):
+ on_date = on_date.isoformat()
+ return Path(vault_root) / events_dir / on_date / name / f"{name}.md"
+
+
+def topic_path(
+ vault_root: str | Path,
+ folder: str,
+ name: str,
+ topics_dir: str = "topics",
+) -> Path:
+ """topics/{folder}/{name}.md under the given vault root."""
+ return Path(vault_root) / topics_dir / folder / f"{name}.md"
+
+
+def is_folder_topic(path: str | Path) -> bool:
+ """True if filename stem == parent directory name (folder note convention)."""
+ p = Path(path)
+ return p.stem == p.parent.name
+
+
+def next_suffixed_stem(taken: Iterable[str], base: str) -> str:
+ """Lowest unused `-N` (N≥2). Returns `base` itself if not taken.
+
+ Used by topic_create / sync when a same-stem (topic) or
+ same-name-on-same-day (event) collision is detected — the suggested
+ suffix is *advisory*; the create still rejects so the agent can pick
+ a domain-specific qualifier (e.g., `Apple-Inc` vs `Apple-Fruit`)
+ that carries more meaning than a numeric suffix.
+
+ Examples (with taken={"BABA", "BABA-2"}):
+ next_suffixed_stem(taken, "BABA") == "BABA-3"
+ next_suffixed_stem(taken, "MSFT") == "MSFT"
+ """
+ taken_set = set(taken)
+ if base not in taken_set:
+ return base
+ pattern = re.compile(rf"^{re.escape(base)}-(\d+)$")
+ used: set[int] = set()
+ for s in taken_set:
+ m = pattern.match(s)
+ if m:
+ try:
+ used.add(int(m.group(1)))
+ except ValueError:
+ continue
+ n = 2
+ while n in used:
+ n += 1
+ return f"{base}-{n}"
diff --git a/reme2/utils/wikilink.py b/reme2/utils/wikilink.py
new file mode 100644
index 00000000..cc0545e8
--- /dev/null
+++ b/reme2/utils/wikilink.py
@@ -0,0 +1,259 @@
+"""Wikilink protocol — Obsidian-compatible bare links + Dataview-style typed edges.
+
+## Bare wikilinks (Obsidian)
+
+ [[Note]] short form (resolve by stem)
+ [[path/to/Note]] path form
+ [[Note#Section]] heading anchor
+ [[Note#^block-id]] block anchor
+ [[Note|Alias]] display alias
+ ![[Note]] embed (transclusion)
+ ![[Note#Section|Alias]] all combined
+
+## Typed wikilinks / property edges (Dataview inline field syntax)
+
+ [predicate:: [[Target]]] typed link (bracketed, visible)
+ (predicate:: [[Target]]) typed link (parenthesized, hidden in render)
+ [predicate:: scalar value] typed scalar (becomes a property, not a graph edge)
+
+`predicate` is identifier-shaped: starts with a letter, then letters /
+digits / `_` / `-`. The link inside follows the bare protocol above.
+
+## Output schema
+
+Wikilink edges are returned as `FileEdge` (the persistence model exposed
+via `file_store.get_edges(path)`). Typed scalar properties are returned
+as `InlineField` — they carry no link target, just `{predicate, value}`.
+
+## Functions
+
+Bare extraction (back-compat — return targets only):
+ extract_wikilinks(text) → list[str]
+ extract_wikilinks_from_metadata(d) → list[str]
+
+Structured parsing (return FileEdge):
+ parse_wikilinks(text) → list[FileEdge]
+ parse_wikilinks_from_metadata(d) → list[FileEdge]
+ extract_inline_fields(text) → list[InlineField]
+ extract_typed_edges(text) → list[FileEdge] (predicate-bearing only)
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+
+from ..schema.file_edge import FileEdge
+
+
+EdgeSource = Literal["regex", "frontmatter", "llm"]
+
+
+# -- Regexes --------------------------------------------------------------
+
+# Bare wikilink. Group(1) = target, kept as the FIRST capturing group so
+# legacy callers using `m.group(1)` (e.g. wikilink rewriters) still work.
+# The leading `(?:!)?` is non-capturing — read the embed prefix off
+# `m.group(0).startswith("!")` instead.
+WIKILINK_RE = re.compile(
+ r"""
+ (?:!)?
+ \[\[
+ (?P[^\]\|\#\n]+?)
+ (?:\#(?P[^\]\|\n]+))?
+ (?:\|(?P[^\]\n]+))?
+ \]\]
+ """,
+ re.VERBOSE,
+)
+
+# Dataview inline field: [pred:: value] or (pred:: value). Value may be a
+# bare wikilink OR a scalar with no inner brackets. The open/close bracket
+# pair isn't enforced by the regex — `_bracket_pair_ok` rejects mismatched
+# `[..)`-style fragments.
+INLINE_FIELD_RE = re.compile(
+ r"""
+ (?P[\[\(])
+ (?P[A-Za-z][\w\-]*)
+ \s*::\s*
+ (?P
+ (?: !? \[\[ [^\]\|\#\n]+? (?:\#[^\]\|\n]+)? (?:\|[^\]\n]+)? \]\] )
+ |
+ (?: [^\]\)\n]+? )
+ )
+ (?P[\]\)])
+ """,
+ re.VERBOSE,
+)
+
+
+# -- Auxiliary schema (typed scalar) --------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class InlineField:
+ """Typed scalar property — `[predicate:: value]` where value is NOT a wikilink.
+
+ Distinct from `FileEdge`: a scalar property doesn't point at another file,
+ so it's not a graph edge — it's an attribute on the source file.
+ """
+
+ predicate: str
+ value: str
+
+
+# -- Internal helpers -----------------------------------------------------
+
+
+def _edge_from_match(
+ m: re.Match,
+ *,
+ predicate: str | None = None,
+ source: EdgeSource = "regex",
+) -> FileEdge:
+ anchor = m.group("anchor")
+ alias = m.group("alias")
+ return FileEdge(
+ target=m.group("target").strip(),
+ anchor=anchor.strip() if anchor else None,
+ alias=alias.strip() if alias else None,
+ embed=m.group(0).startswith("!"),
+ predicate=predicate,
+ source=source,
+ )
+
+
+def _bracket_pair_ok(open_ch: str, close_ch: str) -> bool:
+ return (open_ch == "[" and close_ch == "]") or (open_ch == "(" and close_ch == ")")
+
+
+# -- Public API: bare extraction (back-compat) ----------------------------
+
+
+def extract_wikilinks(text: str) -> list[str]:
+ """Targets-only list of wikilinks found in text (no dedup).
+
+ Includes wikilinks inside typed wrappers — the watcher's graph
+ projection should see every link target, with or without a predicate.
+ Use `parse_wikilinks` if you need the predicate too.
+ """
+ if not text:
+ return []
+ return [m.group("target").strip() for m in WIKILINK_RE.finditer(text)]
+
+
+def extract_wikilinks_from_metadata(metadata: dict) -> list[str]:
+ """Recursively walk a frontmatter dict, extracting wikilink targets from string leaves."""
+ links: list[str] = []
+
+ def _walk(value) -> None:
+ if isinstance(value, str):
+ links.extend(extract_wikilinks(value))
+ elif isinstance(value, dict):
+ for v in value.values():
+ _walk(v)
+ elif isinstance(value, (list, tuple, set)):
+ for item in value:
+ _walk(item)
+
+ _walk(metadata)
+ return links
+
+
+# -- Public API: structured parsing ---------------------------------------
+
+
+def parse_wikilinks(text: str) -> list[FileEdge]:
+ """Structured parse — bare wikilinks AND typed-wrapper predicates.
+
+ Order is by source position. A wikilink inside a typed wrapper
+ appears once, with its predicate attached. All edges have
+ `source="regex"`.
+ """
+ if not text:
+ return []
+
+ typed_spans: list[tuple[int, int]] = []
+ items: list[tuple[int, FileEdge]] = []
+
+ for m in INLINE_FIELD_RE.finditer(text):
+ if not _bracket_pair_ok(m.group("open"), m.group("close")):
+ continue
+ value = m.group("value").strip()
+ wm = WIKILINK_RE.fullmatch(value)
+ if wm is None:
+ continue # scalar — handled by extract_inline_fields
+ items.append((
+ m.start(),
+ _edge_from_match(wm, predicate=m.group("predicate").strip()),
+ ))
+ typed_spans.append(m.span())
+
+ for m in WIKILINK_RE.finditer(text):
+ s, e = m.span()
+ if any(ts <= s and e <= te for ts, te in typed_spans):
+ continue # this bare match is the inner link of a typed wrapper
+ items.append((s, _edge_from_match(m)))
+
+ items.sort(key=lambda pair: pair[0])
+ return [w for _, w in items]
+
+
+def extract_inline_fields(text: str) -> list[InlineField]:
+ """Typed scalar fields — `[pred:: value]` whose value is NOT a wikilink.
+
+ Link-valued typed fields are returned by `parse_wikilinks` instead,
+ since they're graph edges rather than scalar properties.
+ """
+ if not text:
+ return []
+ out: list[InlineField] = []
+ for m in INLINE_FIELD_RE.finditer(text):
+ if not _bracket_pair_ok(m.group("open"), m.group("close")):
+ continue
+ value = m.group("value").strip()
+ if WIKILINK_RE.fullmatch(value) is not None:
+ continue
+ out.append(InlineField(predicate=m.group("predicate").strip(), value=value))
+ return out
+
+
+def extract_typed_edges(text: str) -> list[FileEdge]:
+ """Sugar: just the predicate-bearing wikilinks (typed graph edges)."""
+ return [w for w in parse_wikilinks(text) if w.is_typed]
+
+
+def parse_wikilinks_from_metadata(metadata: dict) -> list[FileEdge]:
+ """Recursive frontmatter parse — frontmatter keys act as predicates.
+
+ `author: "[[John]]"` → `FileEdge(target="John", predicate="author", source="frontmatter")`.
+ `related: ["[[X]]", "[[Y]]"]` → two `FileEdge(predicate="related", ...)`.
+ A bare wikilink in a string value with no parent key keeps `predicate=None`.
+ Inline `[pred:: ...]` syntax inside a string value still wins over the
+ inherited frontmatter key (predicate carried from the inline parse).
+ All edges have `source="frontmatter"`.
+ """
+ links: list[FileEdge] = []
+
+ def _walk(value, predicate: str | None) -> None:
+ if isinstance(value, str):
+ for w in parse_wikilinks(value):
+ effective = w.predicate if w.predicate is not None else predicate
+ links.append(FileEdge(
+ target=w.target,
+ anchor=w.anchor,
+ alias=w.alias,
+ embed=w.embed,
+ predicate=effective,
+ source="frontmatter",
+ ))
+ elif isinstance(value, dict):
+ for k, v in value.items():
+ _walk(v, predicate=str(k))
+ elif isinstance(value, (list, tuple, set)):
+ for item in value:
+ _walk(item, predicate=predicate)
+
+ _walk(metadata, predicate=None)
+ return links
diff --git a/reme2/vector_based/__init__.py b/reme2/vector_based/__init__.py
deleted file mode 100644
index e69de29b..00000000