mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-21 00:22:45 +00:00
up
This commit is contained in:
parent
7cd2838c98
commit
703c43be23
5 changed files with 45 additions and 66 deletions
|
|
@ -36,6 +36,7 @@ class BaseEmbeddingModel(BaseComponent):
|
|||
max_input_length: int = 8192,
|
||||
max_cache_size: int = 2000,
|
||||
enable_cache: bool = True,
|
||||
cache_name: str = "",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -54,7 +55,8 @@ class BaseEmbeddingModel(BaseComponent):
|
|||
self._cache_misses = 0
|
||||
|
||||
self.working_dir = self.app_context.app_config.working_dir if self.app_context is not None else ""
|
||||
self.cache_path: Path = Path(self.working_dir) / "embedding_cache" / f"{self.name}.npz"
|
||||
self.cache_name: str = cache_name or self.name
|
||||
self.cache_path: Path = Path(self.working_dir) / "embedding_cache" / f"{self.cache_name}.npz"
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear in-memory cache and reset statistics."""
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ class BaseFileStore(BaseComponent):
|
|||
def __init__(
|
||||
self,
|
||||
store_name: str,
|
||||
store_path: str | Path,
|
||||
embedding_model: str = "default",
|
||||
fts_enabled: bool = True,
|
||||
**kwargs,
|
||||
|
|
@ -26,30 +25,29 @@ class BaseFileStore(BaseComponent):
|
|||
super().__init__(**kwargs)
|
||||
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
|
||||
raise ValueError(f"Invalid store name '{store_name}'. Only alphanumeric and underscores allowed.")
|
||||
self.store_name = store_name
|
||||
self.store_path = Path(store_path)
|
||||
self.store_path.mkdir(parents=True, exist_ok=True)
|
||||
self.working_dir = self.app_context.app_config.working_dir if self.app_context else ""
|
||||
self.store_name = store_name or self.name
|
||||
self._embedding_model_name = embedding_model
|
||||
self.fts_enabled = fts_enabled
|
||||
|
||||
self.embedding_model: BaseEmbeddingModel | None = None
|
||||
self.vector_enabled = bool(embedding_model)
|
||||
self.fts_enabled = fts_enabled
|
||||
self.working_dir = self.app_context.app_config.working_dir if self.app_context else ""
|
||||
self.store_path = Path(self.working_dir) / "file_store" / store_name
|
||||
self.store_path.mkdir(parents=True, exist_ok=True)
|
||||
if not self.vector_enabled and not self.fts_enabled:
|
||||
raise ValueError("At least one of embedding_model or fts_enabled must be set.")
|
||||
|
||||
# Lifecycle
|
||||
|
||||
async def _start(self) -> None:
|
||||
if not self._embedding_model_name:
|
||||
return
|
||||
assert self.app_context is not None
|
||||
models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
|
||||
if self._embedding_model_name not in models:
|
||||
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
|
||||
model = models[self._embedding_model_name]
|
||||
if not isinstance(model, BaseEmbeddingModel):
|
||||
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
|
||||
self.embedding_model = model
|
||||
if self.vector_enabled and self.app_context is not None:
|
||||
model_dict = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
|
||||
if self._embedding_model_name not in model_dict:
|
||||
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
|
||||
model = model_dict[self._embedding_model_name]
|
||||
if not isinstance(model, BaseEmbeddingModel):
|
||||
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
|
||||
self.embedding_model = model
|
||||
|
||||
async def _close(self) -> None:
|
||||
self.embedding_model = None
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ class LocalFileStore(BaseFileStore):
|
|||
self._encoding = encoding
|
||||
self._nodes: dict[str, FileNode] = {}
|
||||
self._chunks: dict[str, FileChunk] = {}
|
||||
self._nodes_file = self.store_path / f"{self.store_name}_nodes.jsonl"
|
||||
self._chunks_file = self.store_path / f"{self.store_name}_chunks.jsonl"
|
||||
self._nodes_file = self.store_path / "nodes.jsonl"
|
||||
self._chunks_file = self.store_path / "chunks.jsonl"
|
||||
|
||||
# Lifecycle
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,3 @@ class FileChunk(EmbNode):
|
|||
@property
|
||||
def score(self) -> float:
|
||||
return self.scores.get("score", 0.0)
|
||||
|
||||
@property
|
||||
def hash(self) -> str:
|
||||
return "_".join([self.id, self.path, str(self.start_line), str(self.end_line)])
|
||||
|
|
|
|||
|
|
@ -1,51 +1,34 @@
|
|||
"""FileNode — engine-level file index entry.
|
||||
|
||||
Structural fields only: `path`, `st_mtime`, `edges`. Frontmatter fields
|
||||
ride along as Pydantic extras (model_config has `extra="allow"`), so a
|
||||
markdown file's parsed frontmatter dict is flattened into the node:
|
||||
|
||||
FileNode(path=..., st_mtime=..., edges=[...], **frontmatter_dict)
|
||||
|
||||
Two convenience properties:
|
||||
|
||||
* `file` — `Path(self.path).stem`, the filename without extension.
|
||||
* `metadata` — dict view of the extras (the original frontmatter).
|
||||
|
||||
Domain-aware code that wants typed access to memory-schema fields
|
||||
(lifecycle / scope / source / role / status / confidence …) wraps with
|
||||
`reme2.memory.schema.MemoryFileNode.model_validate(node.model_dump())`.
|
||||
|
||||
The engine layer (file_store, file_parser, file_watcher) never imports
|
||||
the memory schema — it stays domain-agnostic and just shuttles
|
||||
`FileNode` instances around with their extras intact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from reme2.schema.file_edge import FileEdge
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
|
||||
class FileNode(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
path: str = Field(...)
|
||||
st_mtime: float = Field(...)
|
||||
edges: list["FileEdge"] = Field(default_factory=list)
|
||||
class FileEdge(BaseModel):
|
||||
link: str = Field(default=...)
|
||||
predicate: str | None = Field(default=None)
|
||||
|
||||
@property
|
||||
def file(self) -> str:
|
||||
"""Filename stem derived from `path` (no extension)."""
|
||||
return Path(self.path).stem
|
||||
def link_path(self) -> str:
|
||||
return self.link.split("#", 1)[0]
|
||||
|
||||
@property
|
||||
def link_anchor(self) -> str:
|
||||
link_split = self.link.split("#", 1)
|
||||
return link_split[1] if len(link_split) > 1 else ""
|
||||
|
||||
|
||||
class FileFrontMatter(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
title: str = Field(default="")
|
||||
description: str = Field(default="")
|
||||
tags: list[str] | None = Field(default=None)
|
||||
|
||||
@property
|
||||
def metadata(self) -> dict:
|
||||
"""Frontmatter dict view — extras stored via `extra="allow"`.
|
||||
|
||||
Returns a fresh copy so callers can't mutate the node's internal
|
||||
state by writing into the dict.
|
||||
"""
|
||||
return dict(self.__pydantic_extra__ or {})
|
||||
|
||||
|
||||
class FileNode(BaseModel):
|
||||
path: str = Field(default=...)
|
||||
st_mtime: float = Field(default=...)
|
||||
edges: list[FileEdge] = Field(default_factory=list)
|
||||
front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue