This commit is contained in:
jinli.yl 2026-05-12 23:49:21 +08:00
parent 7cd2838c98
commit 703c43be23
5 changed files with 45 additions and 66 deletions

View file

@ -36,6 +36,7 @@ class BaseEmbeddingModel(BaseComponent):
max_input_length: int = 8192,
max_cache_size: int = 2000,
enable_cache: bool = True,
cache_name: str = "",
**kwargs,
):
super().__init__(**kwargs)
@ -54,7 +55,8 @@ class BaseEmbeddingModel(BaseComponent):
self._cache_misses = 0
self.working_dir = self.app_context.app_config.working_dir if self.app_context is not None else ""
self.cache_path: Path = Path(self.working_dir) / "embedding_cache" / f"{self.name}.npz"
self.cache_name: str = cache_name or self.name
self.cache_path: Path = Path(self.working_dir) / "embedding_cache" / f"{self.cache_name}.npz"
def clear_cache(self) -> None:
"""Clear in-memory cache and reset statistics."""

View file

@ -18,7 +18,6 @@ class BaseFileStore(BaseComponent):
def __init__(
self,
store_name: str,
store_path: str | Path,
embedding_model: str = "default",
fts_enabled: bool = True,
**kwargs,
@ -26,30 +25,29 @@ class BaseFileStore(BaseComponent):
super().__init__(**kwargs)
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
raise ValueError(f"Invalid store name '{store_name}'. Only alphanumeric and underscores allowed.")
self.store_name = store_name
self.store_path = Path(store_path)
self.store_path.mkdir(parents=True, exist_ok=True)
self.working_dir = self.app_context.app_config.working_dir if self.app_context else ""
self.store_name = store_name or self.name
self._embedding_model_name = embedding_model
self.fts_enabled = fts_enabled
self.embedding_model: BaseEmbeddingModel | None = None
self.vector_enabled = bool(embedding_model)
self.fts_enabled = fts_enabled
self.working_dir = self.app_context.app_config.working_dir if self.app_context else ""
self.store_path = Path(self.working_dir) / "file_store" / store_name
self.store_path.mkdir(parents=True, exist_ok=True)
if not self.vector_enabled and not self.fts_enabled:
raise ValueError("At least one of embedding_model or fts_enabled must be set.")
# Lifecycle
async def _start(self) -> None:
if not self._embedding_model_name:
return
assert self.app_context is not None
models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
if self._embedding_model_name not in models:
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
model = models[self._embedding_model_name]
if not isinstance(model, BaseEmbeddingModel):
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
self.embedding_model = model
if self.vector_enabled and self.app_context is not None:
model_dict = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
if self._embedding_model_name not in model_dict:
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
model = model_dict[self._embedding_model_name]
if not isinstance(model, BaseEmbeddingModel):
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
self.embedding_model = model
async def _close(self) -> None:
self.embedding_model = None

View file

@ -20,8 +20,8 @@ class LocalFileStore(BaseFileStore):
self._encoding = encoding
self._nodes: dict[str, FileNode] = {}
self._chunks: dict[str, FileChunk] = {}
self._nodes_file = self.store_path / f"{self.store_name}_nodes.jsonl"
self._chunks_file = self.store_path / f"{self.store_name}_chunks.jsonl"
self._nodes_file = self.store_path / "nodes.jsonl"
self._chunks_file = self.store_path / "chunks.jsonl"
# Lifecycle

View file

@ -12,7 +12,3 @@ class FileChunk(EmbNode):
@property
def score(self) -> float:
return self.scores.get("score", 0.0)
@property
def hash(self) -> str:
return "_".join([self.id, self.path, str(self.start_line), str(self.end_line)])

View file

@ -1,51 +1,34 @@
"""FileNode — engine-level file index entry.
Structural fields only: `path`, `st_mtime`, `edges`. Frontmatter fields
ride along as Pydantic extras (model_config has `extra="allow"`), so a
markdown file's parsed frontmatter dict is flattened into the node:
FileNode(path=..., st_mtime=..., edges=[...], **frontmatter_dict)
Two convenience properties:
* `file` `Path(self.path).stem`, the filename without extension.
* `metadata` dict view of the extras (the original frontmatter).
Domain-aware code that wants typed access to memory-schema fields
(lifecycle / scope / source / role / status / confidence ) wraps with
`reme2.memory.schema.MemoryFileNode.model_validate(node.model_dump())`.
The engine layer (file_store, file_parser, file_watcher) never imports
the memory schema it stays domain-agnostic and just shuttles
`FileNode` instances around with their extras intact.
"""
from __future__ import annotations
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
from reme2.schema.file_edge import FileEdge
from pydantic import BaseModel, Field, ConfigDict
class FileNode(BaseModel):
model_config = ConfigDict(extra="allow")
path: str = Field(...)
st_mtime: float = Field(...)
edges: list["FileEdge"] = Field(default_factory=list)
class FileEdge(BaseModel):
link: str = Field(default=...)
predicate: str | None = Field(default=None)
@property
def file(self) -> str:
"""Filename stem derived from `path` (no extension)."""
return Path(self.path).stem
def link_path(self) -> str:
return self.link.split("#", 1)[0]
@property
def link_anchor(self) -> str:
link_split = self.link.split("#", 1)
return link_split[1] if len(link_split) > 1 else ""
class FileFrontMatter(BaseModel):
model_config = ConfigDict(extra="allow")
title: str = Field(default="")
description: str = Field(default="")
tags: list[str] | None = Field(default=None)
@property
def metadata(self) -> dict:
"""Frontmatter dict view — extras stored via `extra="allow"`.
Returns a fresh copy so callers can't mutate the node's internal
state by writing into the dict.
"""
return dict(self.__pydantic_extra__ or {})
class FileNode(BaseModel):
path: str = Field(default=...)
st_mtime: float = Field(default=...)
edges: list[FileEdge] = Field(default_factory=list)
front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter)