From 703c43be23c869720f3f062f0cd75065f2dd1bda Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Tue, 12 May 2026 23:49:21 +0800 Subject: [PATCH] up --- .../embedding/base_embedding_model.py | 4 +- reme2/component/file_store/base_file_store.py | 30 ++++---- .../component/file_store/local_file_store.py | 4 +- reme2/schema/file_chunk.py | 4 -- reme2/schema/file_node.py | 69 +++++++------------ 5 files changed, 45 insertions(+), 66 deletions(-) diff --git a/reme2/component/embedding/base_embedding_model.py b/reme2/component/embedding/base_embedding_model.py index 6bc19057..179a28cb 100644 --- a/reme2/component/embedding/base_embedding_model.py +++ b/reme2/component/embedding/base_embedding_model.py @@ -36,6 +36,7 @@ class BaseEmbeddingModel(BaseComponent): max_input_length: int = 8192, max_cache_size: int = 2000, enable_cache: bool = True, + cache_name: str = "", **kwargs, ): super().__init__(**kwargs) @@ -54,7 +55,8 @@ class BaseEmbeddingModel(BaseComponent): self._cache_misses = 0 self.working_dir = self.app_context.app_config.working_dir if self.app_context is not None else "" - self.cache_path: Path = Path(self.working_dir) / "embedding_cache" / f"{self.name}.npz" + self.cache_name: str = cache_name or self.name + self.cache_path: Path = Path(self.working_dir) / "embedding_cache" / f"{self.cache_name}.npz" def clear_cache(self) -> None: """Clear in-memory cache and reset statistics.""" diff --git a/reme2/component/file_store/base_file_store.py b/reme2/component/file_store/base_file_store.py index bda59724..276de1fa 100644 --- a/reme2/component/file_store/base_file_store.py +++ b/reme2/component/file_store/base_file_store.py @@ -18,7 +18,6 @@ class BaseFileStore(BaseComponent): def __init__( self, store_name: str, - store_path: str | Path, embedding_model: str = "default", fts_enabled: bool = True, **kwargs, @@ -26,30 +25,29 @@ class BaseFileStore(BaseComponent): super().__init__(**kwargs) if not re.match(r"^[a-zA-Z0-9_]+$", store_name): raise ValueError(f"Invalid store name '{store_name}'. Only alphanumeric and underscores allowed.") - self.store_name = store_name - self.store_path = Path(store_path) - self.store_path.mkdir(parents=True, exist_ok=True) - self.working_dir = self.app_context.app_config.working_dir if self.app_context else "" + self.store_name = store_name or self.name self._embedding_model_name = embedding_model + self.fts_enabled = fts_enabled + self.embedding_model: BaseEmbeddingModel | None = None self.vector_enabled = bool(embedding_model) - self.fts_enabled = fts_enabled + self.working_dir = self.app_context.app_config.working_dir if self.app_context else "" + self.store_path = Path(self.working_dir) / "file_store" / store_name + self.store_path.mkdir(parents=True, exist_ok=True) if not self.vector_enabled and not self.fts_enabled: raise ValueError("At least one of embedding_model or fts_enabled must be set.") # Lifecycle async def _start(self) -> None: - if not self._embedding_model_name: - return - assert self.app_context is not None - models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {}) - if self._embedding_model_name not in models: - raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.") - model = models[self._embedding_model_name] - if not isinstance(model, BaseEmbeddingModel): - raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}") - self.embedding_model = model + if self.vector_enabled and self.app_context is not None: + model_dict = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {}) + if self._embedding_model_name not in model_dict: + raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.") + model = model_dict[self._embedding_model_name] + if not isinstance(model, BaseEmbeddingModel): + raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}") + self.embedding_model = model async def _close(self) -> None: self.embedding_model = None diff --git a/reme2/component/file_store/local_file_store.py b/reme2/component/file_store/local_file_store.py index c16d3f0d..907d72d6 100644 --- a/reme2/component/file_store/local_file_store.py +++ b/reme2/component/file_store/local_file_store.py @@ -20,8 +20,8 @@ class LocalFileStore(BaseFileStore): self._encoding = encoding self._nodes: dict[str, FileNode] = {} self._chunks: dict[str, FileChunk] = {} - self._nodes_file = self.store_path / f"{self.store_name}_nodes.jsonl" - self._chunks_file = self.store_path / f"{self.store_name}_chunks.jsonl" + self._nodes_file = self.store_path / "nodes.jsonl" + self._chunks_file = self.store_path / "chunks.jsonl" # Lifecycle diff --git a/reme2/schema/file_chunk.py b/reme2/schema/file_chunk.py index 1b46addc..3ccb7103 100644 --- a/reme2/schema/file_chunk.py +++ b/reme2/schema/file_chunk.py @@ -12,7 +12,3 @@ class FileChunk(EmbNode): @property def score(self) -> float: return self.scores.get("score", 0.0) - - @property - def hash(self) -> str: - return "_".join([self.id, self.path, str(self.start_line), str(self.end_line)]) diff --git a/reme2/schema/file_node.py b/reme2/schema/file_node.py index b3f11a47..73d34dc4 100644 --- a/reme2/schema/file_node.py +++ b/reme2/schema/file_node.py @@ -1,51 +1,34 @@ -"""FileNode — engine-level file index entry. - -Structural fields only: `path`, `st_mtime`, `edges`. Frontmatter fields -ride along as Pydantic extras (model_config has `extra="allow"`), so a -markdown file's parsed frontmatter dict is flattened into the node: - - FileNode(path=..., st_mtime=..., edges=[...], **frontmatter_dict) - -Two convenience properties: - - * `file` — `Path(self.path).stem`, the filename without extension. - * `metadata` — dict view of the extras (the original frontmatter). - -Domain-aware code that wants typed access to memory-schema fields -(lifecycle / scope / source / role / status / confidence …) wraps with -`reme2.memory.schema.MemoryFileNode.model_validate(node.model_dump())`. - -The engine layer (file_store, file_parser, file_watcher) never imports -the memory schema — it stays domain-agnostic and just shuttles -`FileNode` instances around with their extras intact. -""" - -from __future__ import annotations - -from pathlib import Path - -from pydantic import BaseModel, ConfigDict, Field - -from reme2.schema.file_edge import FileEdge +from pydantic import BaseModel, Field, ConfigDict -class FileNode(BaseModel): - model_config = ConfigDict(extra="allow") - - path: str = Field(...) - st_mtime: float = Field(...) - edges: list["FileEdge"] = Field(default_factory=list) +class FileEdge(BaseModel): + link: str = Field(default=...) + predicate: str | None = Field(default=None) @property - def file(self) -> str: - """Filename stem derived from `path` (no extension).""" - return Path(self.path).stem + def link_path(self) -> str: + return self.link.split("#", 1)[0] + + @property + def link_anchor(self) -> str: + link_split = self.link.split("#", 1) + return link_split[1] if len(link_split) > 1 else "" + + +class FileFrontMatter(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str = Field(default="") + description: str = Field(default="") + tags: list[str] | None = Field(default=None) @property def metadata(self) -> dict: - """Frontmatter dict view — extras stored via `extra="allow"`. - - Returns a fresh copy so callers can't mutate the node's internal - state by writing into the dict. - """ return dict(self.__pydantic_extra__ or {}) + + +class FileNode(BaseModel): + path: str = Field(default=...) + st_mtime: float = Field(default=...) + edges: list[FileEdge] = Field(default_factory=list) + front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter)