mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-21 00:22:45 +00:00
up
This commit is contained in:
parent
3a573d9122
commit
dacbf52637
5 changed files with 15 additions and 22 deletions
|
|
@ -16,7 +16,6 @@ class BaseFileStore(BaseComponent):
|
|||
store_name: str,
|
||||
embedding_model: str = "default",
|
||||
keyword_index: str = "default",
|
||||
fts_enabled: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -25,30 +24,26 @@ class BaseFileStore(BaseComponent):
|
|||
self.store_name = store_name or self.name
|
||||
self._embedding_model_name = embedding_model
|
||||
self._keyword_index_name = keyword_index
|
||||
self.fts_enabled = fts_enabled
|
||||
|
||||
self.embedding_model: BaseEmbeddingModel | None = None
|
||||
self.keyword_index: BaseKeywordIndex | None = None
|
||||
self.vector_enabled = bool(embedding_model)
|
||||
self.store_path = self.working_path / self.component_type.value / store_name
|
||||
self.store_path.mkdir(parents=True, exist_ok=True)
|
||||
if not self.vector_enabled and not self.fts_enabled:
|
||||
raise ValueError("At least one of embedding_model or fts_enabled must be set.")
|
||||
if not embedding_model and not keyword_index:
|
||||
raise ValueError("At least one of embedding_model or keyword_index must be set.")
|
||||
|
||||
self.file_nodes: dict[str, FileNode] = {}
|
||||
|
||||
async def _start(self) -> None:
|
||||
if self.vector_enabled:
|
||||
if self._embedding_model_name:
|
||||
self.embedding_model = self.get_component(ComponentEnum.EMBEDDING_MODEL, self._embedding_model_name)
|
||||
if self.fts_enabled:
|
||||
if self._keyword_index_name:
|
||||
self.keyword_index = self.get_component(ComponentEnum.KEYWORD_INDEX, self._keyword_index_name)
|
||||
await self.load_file_nodes()
|
||||
|
||||
async def _close(self) -> None:
|
||||
if self.vector_enabled:
|
||||
self.embedding_model = None
|
||||
if self.fts_enabled:
|
||||
self.keyword_index = None
|
||||
self.embedding_model = None
|
||||
self.keyword_index = None
|
||||
await self.dump_file_nodes()
|
||||
|
||||
async def load_file_nodes(self):
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ class LocalFileStore(BaseFileStore):
|
|||
for node, chunks in file:
|
||||
old_node = self.file_nodes.pop(node.path, None)
|
||||
cached = {}
|
||||
if old_node and self.vector_enabled:
|
||||
if old_node and self.embedding_model:
|
||||
for cid in old_node.chunk_ids:
|
||||
old = self.file_chunks.pop(cid, None)
|
||||
if old and old.embedding:
|
||||
|
|
@ -88,7 +88,7 @@ class LocalFileStore(BaseFileStore):
|
|||
node.chunk_ids = []
|
||||
needs_embed = []
|
||||
for c in chunks:
|
||||
if self.vector_enabled and not c.embedding:
|
||||
if self.embedding_model and not c.embedding:
|
||||
if c.id in cached:
|
||||
c.embedding = cached[c.id]
|
||||
elif c.text:
|
||||
|
|
@ -100,7 +100,7 @@ class LocalFileStore(BaseFileStore):
|
|||
if needs_embed and self.embedding_model:
|
||||
await self.embedding_model.get_node_embeddings(needs_embed)
|
||||
|
||||
if self.fts_enabled and self.keyword_index:
|
||||
if self.keyword_index:
|
||||
await self.keyword_index.add_docs({c.id: c.text for c in chunks if c.text})
|
||||
|
||||
async def delete_by_path(self, path: str | list[str]) -> None:
|
||||
|
|
@ -114,13 +114,13 @@ class LocalFileStore(BaseFileStore):
|
|||
self.file_chunks.pop(cid, None)
|
||||
deleted_chunk_ids.append(cid)
|
||||
|
||||
if self.fts_enabled and self.keyword_index and deleted_chunk_ids:
|
||||
if self.keyword_index and deleted_chunk_ids:
|
||||
await self.keyword_index.delete_docs(deleted_chunk_ids)
|
||||
|
||||
async def clear(self) -> None:
|
||||
self.file_nodes.clear()
|
||||
self.file_chunks.clear()
|
||||
if self.fts_enabled and self.keyword_index:
|
||||
if self.keyword_index:
|
||||
await self.keyword_index.clear()
|
||||
|
||||
# Search
|
||||
|
|
@ -148,7 +148,7 @@ class LocalFileStore(BaseFileStore):
|
|||
return results[:limit]
|
||||
|
||||
async def keyword_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
|
||||
if not self.fts_enabled or self.keyword_index is None:
|
||||
if not self.keyword_index:
|
||||
return []
|
||||
|
||||
query = query.strip()
|
||||
|
|
|
|||
|
|
@ -364,7 +364,6 @@ components:
|
|||
store_name: "reme"
|
||||
db_path: "./vault/.reme"
|
||||
working_dir: "./vault"
|
||||
fts_enabled: true
|
||||
|
||||
file_watcher:
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -257,7 +257,6 @@ components:
|
|||
store_name: "reme"
|
||||
db_path: "./vault/.reme"
|
||||
working_dir: "./vault"
|
||||
fts_enabled: true
|
||||
|
||||
file_watcher:
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ class HybridRetriever(BaseRetriever):
|
|||
candidates = min(200, max(1, int(max_results * self.candidate_multiplier)))
|
||||
text_weight = 1.0 - self.vector_weight
|
||||
|
||||
if fs.vector_enabled and fs.fts_enabled:
|
||||
if fs.embedding_model and fs.keyword_index:
|
||||
v_task = memory_io.search_vector(fs, query, limit=candidates, chunk_filter=chunk_filter)
|
||||
k_task = memory_io.search_keyword(fs, query, limit=candidates, chunk_filter=chunk_filter)
|
||||
v_results, k_results = await asyncio.gather(v_task, k_task)
|
||||
|
|
@ -185,9 +185,9 @@ class HybridRetriever(BaseRetriever):
|
|||
results = self._merge_vk(
|
||||
v_results, k_results, self.vector_weight, text_weight,
|
||||
)[:max_results]
|
||||
elif fs.vector_enabled:
|
||||
elif fs.embedding_model:
|
||||
results = await memory_io.search_vector(fs, query, limit=max_results, chunk_filter=chunk_filter)
|
||||
elif fs.fts_enabled:
|
||||
elif fs.keyword_index:
|
||||
results = await memory_io.search_keyword(fs, query, limit=max_results, chunk_filter=chunk_filter)
|
||||
else:
|
||||
results = []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue