From c583e06f3ed4fe2363011037597ea5dcd6a30bb1 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Sat, 16 May 2026 23:29:03 +0800 Subject: [PATCH] up --- docs4/todo.md | 4 +++- .../embedding/base_embedding_model.py | 22 ++++++++++++++++++- .../components/file_graph/base_file_graph.py | 3 ++- .../components/file_graph/local_file_graph.py | 2 +- reme4/components/file_graph/nx_file_graph.py | 2 +- .../components/file_store/base_file_store.py | 17 ++++++++++++++ .../components/file_store/local_file_store.py | 13 ++++++++--- .../keyword_index/base_keyword_index.py | 5 +++-- reme4/config/default.yaml | 8 ++++--- reme4/schema/application_config.py | 2 +- 10 files changed, 64 insertions(+), 14 deletions(-) diff --git a/docs4/todo.md b/docs4/todo.md index eca1816c..f98bb09a 100644 --- a/docs4/todo.md +++ b/docs4/todo.md @@ -4,4 +4,6 @@ 4. error 5. meta信息存在一个地方 6. 测试一个完整的Service client的框架，测试各种命令 -7. config 默认改成default \ No newline at end of file +7. config 默认改成default +8. todo reindex +9. \ No newline at end of file diff --git a/reme4/components/embedding/base_embedding_model.py b/reme4/components/embedding/base_embedding_model.py index 48aca850..4fce0535 100644 --- a/reme4/components/embedding/base_embedding_model.py +++ b/reme4/components/embedding/base_embedding_model.py @@ -30,6 +30,7 @@ class BaseEmbeddingModel(BaseComponent): max_input_length: int = 8192, max_cache_size: int = 10000, enable_cache: bool = True, + cache_version: str = "v1", max_retries: int = 3, **kwargs, ): @@ -43,19 +44,38 @@ class BaseEmbeddingModel(BaseComponent): self.max_input_length = max_input_length self.max_cache_size = max_cache_size self.enable_cache = enable_cache + self.cache_version = cache_version self.max_retries = max_retries self._embedding_cache: OrderedDict[str, np.ndarray] = OrderedDict() + self.is_healthy: bool = True @property def cache_path(self) -> Path: """Disk path for the embedding cache file.""" - return self.working_metadata_path / "embedding_cache" / f"{self.name}.npz" + return self.working_metadata_path / "embedding_cache" / f"{self.name}_{self.cache_version}.npz" async def _start(self) -> None: """Load cache from disk on startup.""" self._embedding_cache.clear() self._load_cache() + async def health_check(self, timeout: float = 2.0) -> bool: + """Probe the provider; sets and returns is_healthy.""" + tag = f"[EMBEDDING HEALTH CHECK] name={self.name} model={self.model_name}" + try: + result = await asyncio.wait_for(self._get_embeddings(["ping"]), timeout=timeout) + if not result or result[0] is None: + raise RuntimeError("empty embedding") + self.is_healthy = True + self.logger.info(f"{tag} -> OK") + except asyncio.TimeoutError: + self.is_healthy = False + self.logger.error(f"{tag} -> FAIL timeout({timeout}s)") + except Exception as e: + self.is_healthy = False + self.logger.error(f"{tag} -> FAIL {type(e).__name__}: {e}") + return self.is_healthy + async def _close(self) -> None: """Persist cache to disk on shutdown.""" self._save_cache() diff --git a/reme4/components/file_graph/base_file_graph.py b/reme4/components/file_graph/base_file_graph.py index 6844c7f5..333d75db 100644 --- a/reme4/components/file_graph/base_file_graph.py +++ b/reme4/components/file_graph/base_file_graph.py @@ -13,9 +13,10 @@ class BaseFileGraph(BaseComponent): component_type = ComponentEnum.FILE_GRAPH - def __init__(self, graph_name: str = "default", **kwargs): + def __init__(self, graph_name: str = "default", graph_version: str = "v1", **kwargs): super().__init__(**kwargs) self.graph_name: str = graph_name or self.name + self.graph_version: str = graph_version self.graph_path: Path = self.working_metadata_path / self.component_type.value self.graph_path.mkdir(parents=True, exist_ok=True) diff --git a/reme4/components/file_graph/local_file_graph.py b/reme4/components/file_graph/local_file_graph.py index b686224e..8aed20d7 100644 --- a/reme4/components/file_graph/local_file_graph.py +++ b/reme4/components/file_graph/local_file_graph.py @@ -16,7 +16,7 @@ class LocalFileGraph(BaseFileGraph): self._nodes: dict[str, FileNode] = {} self._inverse: dict[str, set[str]] = {} # target → {sources} self._pending: dict[str, set[str]] = {} # virtual target → {sources} - self._graph_file: Path = self.graph_path / f"{self.graph_name}.jsonl" + self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.jsonl" # -- Lifecycle --------------------------------------------------------- diff --git a/reme4/components/file_graph/nx_file_graph.py b/reme4/components/file_graph/nx_file_graph.py index f4f4ea13..81695dec 100644 --- a/reme4/components/file_graph/nx_file_graph.py +++ b/reme4/components/file_graph/nx_file_graph.py @@ -22,7 +22,7 @@ class NxFileGraph(BaseFileGraph): if nx is None: raise ImportError("NxFileGraph requires networkx — pip install networkx") self._graph: nx.MultiDiGraph = nx.MultiDiGraph() - self._graph_file: Path = self.graph_path / f"{self.graph_name}.pkl" + self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.pkl" # -- Lifecycle --------------------------------------------------------- diff --git a/reme4/components/file_store/base_file_store.py b/reme4/components/file_store/base_file_store.py index 768e26bb..770d043e 100644 --- a/reme4/components/file_store/base_file_store.py +++ b/reme4/components/file_store/base_file_store.py @@ -21,6 +21,7 @@ class BaseFileStore(BaseComponent): embedding_model: str = "default", keyword_index: str = "default", file_graph: str = "default", + store_version: str = "v1", **kwargs, ): super().__init__(**kwargs) @@ -29,6 +30,7 @@ class BaseFileStore(BaseComponent): from ..keyword_index import BM25Index self.store_name = store_name or self.name + self.store_version = store_version if not embedding_model and not keyword_index: raise ValueError("At least one of embedding_model or keyword_index must be set.") @@ -38,6 +40,21 @@ class BaseFileStore(BaseComponent): self.store_path = self.working_metadata_path / self.component_type.value / store_name self.store_path.mkdir(parents=True, exist_ok=True) + async def _start(self) -> None: + """Probe embedding model; disable vector capability if it fails.""" + if self.embedding_model is None: + return + if not await self.embedding_model.health_check(): + self.logger.warning(f"{self.store_name}: embedding unhealthy, vector disabled") + self.embedding_model = None + + def _disable_embedding(self, reason: str) -> None: + """Drop embedding after a runtime failure; keyword search still works.""" + if self.embedding_model is None: + return + self.logger.error(f"{self.store_name}: embedding disabled, {reason}") + self.embedding_model = None + async def upsert_file( self, file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]], diff --git a/reme4/components/file_store/local_file_store.py b/reme4/components/file_store/local_file_store.py index a16fb2ef..63638dd6 100644 --- a/reme4/components/file_store/local_file_store.py +++ b/reme4/components/file_store/local_file_store.py @@ -17,7 +17,7 @@ class LocalFileStore(BaseFileStore): super().__init__(**kwargs) self.encoding = encoding self.file_chunks: dict[str, FileChunk] = {} - self.chunks_path = self.store_path / "file_chunks.jsonl" + self.chunks_path = self.store_path / f"file_chunks_{self.store_version}.jsonl" # Lifecycle @@ -86,7 +86,10 @@ class LocalFileStore(BaseFileStore): await self.file_graph.upsert_nodes(new_nodes) if needs_embed and self.embedding_model: - await self.embedding_model.get_node_embeddings(needs_embed) + try: + await self.embedding_model.get_node_embeddings(needs_embed) + except Exception as e: + self._disable_embedding(f"upsert: {type(e).__name__}: {e}") if self.keyword_index and keyword_docs: await self.keyword_index.add_docs(keyword_docs) @@ -119,7 +122,11 @@ class LocalFileStore(BaseFileStore): if self.embedding_model is None or not query: return [] - query_embedding = await self.embedding_model.get_embedding(query) + try: + query_embedding = await self.embedding_model.get_embedding(query) + except Exception as e: + self._disable_embedding(f"search: {type(e).__name__}: {e}") + return [] if query_embedding is None: return [] diff --git a/reme4/components/keyword_index/base_keyword_index.py b/reme4/components/keyword_index/base_keyword_index.py index 2791bf83..45681a4d 100644 --- a/reme4/components/keyword_index/base_keyword_index.py +++ b/reme4/components/keyword_index/base_keyword_index.py @@ -13,11 +13,12 @@ class BaseKeywordIndex(BaseComponent): component_type = ComponentEnum.KEYWORD_INDEX - def __init__(self, tokenizer: str = "default", **kwargs): + def __init__(self, tokenizer: str = "default", index_version: str = "v1", **kwargs): super().__init__(**kwargs) from ..tokenizer import RegexTokenizer self.tokenizer = self.bind(tokenizer, BaseTokenizer, default_factory=RegexTokenizer) + self.index_version = index_version self.index_path = self.working_metadata_path / self.component_type.value self.index_path.mkdir(parents=True, exist_ok=True) @@ -38,7 +39,7 @@ class BaseKeywordIndex(BaseComponent): if self.tokenizer is None: raise RuntimeError("Tokenizer not initialized. Call start() first.") name = type(self.tokenizer).__name__.replace("Tokenizer", "").lower() - return self.index_path / f"bm25_{name}.pkl" + return self.index_path / f"bm25_{name}_{self.index_version}.pkl" def _tokenize(self, text: str) -> list[str]: """Tokenize a text string into tokens.""" diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index 59a938ee..1187f491 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -32,8 +32,8 @@ components: embedding_model: default: backend: openai - model_name: text-embedding-3-small - dimensions: 1536 + model_name: text-embedding-v4 + dimensions: 1024 # 3. file_graph — no dependencies file_graph: @@ -65,6 +65,8 @@ components: file_watcher: default: backend: lite - watch_paths: "." + watch_paths: + - MEMORY.md + - memory file_store: default file_parser: default \ No newline at end of file diff --git a/reme4/schema/application_config.py b/reme4/schema/application_config.py index c296a83b..7b1bbda8 100644 --- a/reme4/schema/application_config.py +++ b/reme4/schema/application_config.py @@ -29,7 +29,7 @@ class ApplicationConfig(BaseModel): app_name: str = Field(default=os.getenv("APP_NAME", "ReMe"), description="Application display name") working_dir: str = Field(default=".reme", description="Working directory for runtime files") - metadata_dir: str = Field(default="reme_metadata",description="Subdirectory for ReMe persistent state") + metadata_dir: str = Field(default="reme_metadata", description="Subdirectory for ReMe persistent state") enable_logo: bool = Field(default=True, description="Show ASCII logo on startup") language: str = Field(default="", description="Default language for LLM interactions") log_to_console: bool = Field(default=True, description="Log to console")