up

docs4/todo.md

@@ -4,4 +4,6 @@
 4. error
 5. meta信息存在一个地方
 6. 测试一个完整的Service client的框架，测试各种命令
-7. config 默认改成default
+7. config 默认改成default
+8. todo reindex
+9.

reme4/components/embedding/base_embedding_model.py

@@ -30,6 +30,7 @@ class BaseEmbeddingModel(BaseComponent):
         max_input_length: int = 8192,
         max_cache_size: int = 10000,
         enable_cache: bool = True,
+        cache_version: str = "v1",
         max_retries: int = 3,
         **kwargs,
     ):
@@ -43,19 +44,38 @@ class BaseEmbeddingModel(BaseComponent):
         self.max_input_length = max_input_length
         self.max_cache_size = max_cache_size
         self.enable_cache = enable_cache
+        self.cache_version = cache_version
         self.max_retries = max_retries
         self._embedding_cache: OrderedDict[str, np.ndarray] = OrderedDict()
+        self.is_healthy: bool = True
 
     @property
     def cache_path(self) -> Path:
         """Disk path for the embedding cache file."""
-        return self.working_metadata_path / "embedding_cache" / f"{self.name}.npz"
+        return self.working_metadata_path / "embedding_cache" / f"{self.name}_{self.cache_version}.npz"
 
     async def _start(self) -> None:
         """Load cache from disk on startup."""
         self._embedding_cache.clear()
         self._load_cache()
 
+    async def health_check(self, timeout: float = 2.0) -> bool:
+        """Probe the provider; sets and returns is_healthy."""
+        tag = f"[EMBEDDING HEALTH CHECK] name={self.name} model={self.model_name}"
+        try:
+            result = await asyncio.wait_for(self._get_embeddings(["ping"]), timeout=timeout)
+            if not result or result[0] is None:
+                raise RuntimeError("empty embedding")
+            self.is_healthy = True
+            self.logger.info(f"{tag} -> OK")
+        except asyncio.TimeoutError:
+            self.is_healthy = False
+            self.logger.error(f"{tag} -> FAIL timeout({timeout}s)")
+        except Exception as e:
+            self.is_healthy = False
+            self.logger.error(f"{tag} -> FAIL {type(e).__name__}: {e}")
+        return self.is_healthy
+
     async def _close(self) -> None:
         """Persist cache to disk on shutdown."""
         self._save_cache()

reme4/components/file_graph/base_file_graph.py

@@ -13,9 +13,10 @@ class BaseFileGraph(BaseComponent):
 
     component_type = ComponentEnum.FILE_GRAPH
 
-    def __init__(self, graph_name: str = "default", **kwargs):
+    def __init__(self, graph_name: str = "default", graph_version: str = "v1", **kwargs):
         super().__init__(**kwargs)
         self.graph_name: str = graph_name or self.name
+        self.graph_version: str = graph_version
         self.graph_path: Path = self.working_metadata_path / self.component_type.value
         self.graph_path.mkdir(parents=True, exist_ok=True)
 

reme4/components/file_graph/local_file_graph.py

@@ -16,7 +16,7 @@ class LocalFileGraph(BaseFileGraph):
         self._nodes: dict[str, FileNode] = {}
         self._inverse: dict[str, set[str]] = {}  # target → {sources}
         self._pending: dict[str, set[str]] = {}  # virtual target → {sources}
-        self._graph_file: Path = self.graph_path / f"{self.graph_name}.jsonl"
+        self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.jsonl"
 
     # -- Lifecycle ---------------------------------------------------------
 

reme4/components/file_graph/nx_file_graph.py

@@ -22,7 +22,7 @@ class NxFileGraph(BaseFileGraph):
         if nx is None:
             raise ImportError("NxFileGraph requires networkx — pip install networkx")
         self._graph: nx.MultiDiGraph = nx.MultiDiGraph()
-        self._graph_file: Path = self.graph_path / f"{self.graph_name}.pkl"
+        self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.pkl"
 
     # -- Lifecycle ---------------------------------------------------------
 

reme4/components/file_store/base_file_store.py

@@ -21,6 +21,7 @@ class BaseFileStore(BaseComponent):
         embedding_model: str = "default",
         keyword_index: str = "default",
         file_graph: str = "default",
+        store_version: str = "v1",
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -29,6 +30,7 @@ class BaseFileStore(BaseComponent):
         from ..keyword_index import BM25Index
 
         self.store_name = store_name or self.name
+        self.store_version = store_version
         if not embedding_model and not keyword_index:
             raise ValueError("At least one of embedding_model or keyword_index must be set.")
 
@@ -38,6 +40,21 @@ class BaseFileStore(BaseComponent):
         self.store_path = self.working_metadata_path / self.component_type.value / store_name
         self.store_path.mkdir(parents=True, exist_ok=True)
 
+    async def _start(self) -> None:
+        """Probe embedding model; disable vector capability if it fails."""
+        if self.embedding_model is None:
+            return
+        if not await self.embedding_model.health_check():
+            self.logger.warning(f"{self.store_name}: embedding unhealthy, vector disabled")
+            self.embedding_model = None
+
+    def _disable_embedding(self, reason: str) -> None:
+        """Drop embedding after a runtime failure; keyword search still works."""
+        if self.embedding_model is None:
+            return
+        self.logger.error(f"{self.store_name}: embedding disabled, {reason}")
+        self.embedding_model = None
+
     async def upsert_file(
         self,
         file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],

reme4/components/file_store/local_file_store.py

@@ -17,7 +17,7 @@ class LocalFileStore(BaseFileStore):
         super().__init__(**kwargs)
         self.encoding = encoding
         self.file_chunks: dict[str, FileChunk] = {}
-        self.chunks_path = self.store_path / "file_chunks.jsonl"
+        self.chunks_path = self.store_path / f"file_chunks_{self.store_version}.jsonl"
 
     # Lifecycle
 
@@ -86,7 +86,10 @@ class LocalFileStore(BaseFileStore):
 
         await self.file_graph.upsert_nodes(new_nodes)
         if needs_embed and self.embedding_model:
-            await self.embedding_model.get_node_embeddings(needs_embed)
+            try:
+                await self.embedding_model.get_node_embeddings(needs_embed)
+            except Exception as e:
+                self._disable_embedding(f"upsert: {type(e).__name__}: {e}")
         if self.keyword_index and keyword_docs:
             await self.keyword_index.add_docs(keyword_docs)
 
@@ -119,7 +122,11 @@ class LocalFileStore(BaseFileStore):
         if self.embedding_model is None or not query:
             return []
 
-        query_embedding = await self.embedding_model.get_embedding(query)
+        try:
+            query_embedding = await self.embedding_model.get_embedding(query)
+        except Exception as e:
+            self._disable_embedding(f"search: {type(e).__name__}: {e}")
+            return []
         if query_embedding is None:
             return []
 

reme4/components/keyword_index/base_keyword_index.py

@@ -13,11 +13,12 @@ class BaseKeywordIndex(BaseComponent):
 
     component_type = ComponentEnum.KEYWORD_INDEX
 
-    def __init__(self, tokenizer: str = "default", **kwargs):
+    def __init__(self, tokenizer: str = "default", index_version: str = "v1", **kwargs):
         super().__init__(**kwargs)
         from ..tokenizer import RegexTokenizer
 
         self.tokenizer = self.bind(tokenizer, BaseTokenizer, default_factory=RegexTokenizer)
+        self.index_version = index_version
         self.index_path = self.working_metadata_path / self.component_type.value
         self.index_path.mkdir(parents=True, exist_ok=True)
 
@@ -38,7 +39,7 @@ class BaseKeywordIndex(BaseComponent):
         if self.tokenizer is None:
             raise RuntimeError("Tokenizer not initialized. Call start() first.")
         name = type(self.tokenizer).__name__.replace("Tokenizer", "").lower()
-        return self.index_path / f"bm25_{name}.pkl"
+        return self.index_path / f"bm25_{name}_{self.index_version}.pkl"
 
     def _tokenize(self, text: str) -> list[str]:
         """Tokenize a text string into tokens."""

reme4/config/default.yaml

@@ -32,8 +32,8 @@ components:
   embedding_model:
     default:
       backend: openai
-      model_name: text-embedding-3-small
-      dimensions: 1536
+      model_name: text-embedding-v4
+      dimensions: 1024
 
   # 3. file_graph — no dependencies
   file_graph:
@@ -65,6 +65,8 @@ components:
   file_watcher:
     default:
       backend: lite
-      watch_paths: "."
+      watch_paths:
+        - MEMORY.md
+        - memory
       file_store: default
       file_parser: default

reme4/schema/application_config.py

@@ -29,7 +29,7 @@ class ApplicationConfig(BaseModel):
 
     app_name: str = Field(default=os.getenv("APP_NAME", "ReMe"), description="Application display name")
     working_dir: str = Field(default=".reme", description="Working directory for runtime files")
-    metadata_dir: str = Field(default="reme_metadata",description="Subdirectory for ReMe persistent state")
+    metadata_dir: str = Field(default="reme_metadata", description="Subdirectory for ReMe persistent state")
     enable_logo: bool = Field(default=True, description="Show ASCII logo on startup")
     language: str = Field(default="", description="Default language for LLM interactions")
     log_to_console: bool = Field(default=True, description="Log to console")