mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
514bf35050
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled
docs: add ReMe2 architecture design documentation - Add comprehensive design document (reme2.md) detailing the three-layer architecture (L1/L2/L3) for the vault system - Document new protocols for folder notes and memory management - Specify interface contracts for memory_* and vault_* tools - Outline implementation phases from current state to target refactor: fix typo in personal retriever class - Correct spelling error: 'retri eved_nodes' -> 'retrieved_nodes' in PersonalRetriever.result assignment chore: update gitignore with vault-related patterns - Add '/vault' to ignore vault directory - Add '/reme-plugin' to ignore plugin files - Add '/reme2/vault' to ignore new vault implementation ```
98 lines
3.6 KiB
Python
98 lines
3.6 KiB
Python
"""Abstract base class for file parsers.
|
|
|
|
Single-pass parser: `parse(path, existing_chunks)` returns a `ParsedFile`
|
|
carrying metadata + chunks (with embeddings attached) + edges.
|
|
|
|
The parser owns the embedding pipeline. To avoid re-embedding unchanged
|
|
blocks, the watcher passes in the file's prior chunks; the parser
|
|
hash-diffs and only calls the embedding API for blocks whose hash is
|
|
new. Vanished hashes simply drop out (the file_store's upsert is a
|
|
delete-and-insert).
|
|
|
|
If `existing_chunks` is None / empty, every chunk is embedded fresh.
|
|
"""
|
|
|
|
from abc import abstractmethod
|
|
|
|
from ..base_component import BaseComponent
|
|
from ..embedding import BaseEmbeddingModel
|
|
from ...enumeration import ComponentEnum, FileSuffixEnum
|
|
from ...schema import FileChunk, ParsedFile
|
|
|
|
|
|
class BaseFileParser(BaseComponent):
|
|
"""Single-pass parser producing an embedded `ParsedFile`."""
|
|
|
|
component_type = ComponentEnum.FILE_PARSER
|
|
suffixes: list[FileSuffixEnum] = []
|
|
|
|
def __init__(self, embedding_model: str = "", **kwargs):
|
|
super().__init__(**kwargs)
|
|
self._embedding_model_name: str = embedding_model
|
|
self.embedding_model: BaseEmbeddingModel | None = None
|
|
|
|
async def _start(self) -> None:
|
|
if not self._embedding_model_name:
|
|
return
|
|
assert self.app_context is not None, "app_context must be provided"
|
|
models = self.app_context.components.get(ComponentEnum.EMBEDDING_MODEL, {})
|
|
if self._embedding_model_name not in models:
|
|
raise ValueError(f"Embedding model '{self._embedding_model_name}' not found.")
|
|
model = models[self._embedding_model_name]
|
|
if not isinstance(model, BaseEmbeddingModel):
|
|
raise TypeError(f"Expected BaseEmbeddingModel, got {type(model).__name__}")
|
|
self.embedding_model = model
|
|
|
|
async def _close(self) -> None:
|
|
self.embedding_model = None
|
|
|
|
async def _embed_chunks(self, chunks: list[FileChunk]) -> list[FileChunk]:
|
|
"""Attach embeddings to chunks if an embedding model is configured.
|
|
|
|
Failures (missing embeddings) leave `chunk.embedding=None`; the
|
|
file_store can still keyword-search them.
|
|
"""
|
|
if not chunks or self.embedding_model is None:
|
|
return chunks
|
|
try:
|
|
await self.embedding_model.get_node_embeddings(chunks)
|
|
except Exception as e:
|
|
self.logger.warning(f"embedding chunks failed: {e}")
|
|
return chunks
|
|
|
|
@staticmethod
|
|
def _hash_diff_attach(
|
|
chunks: list[FileChunk],
|
|
existing_chunks: list[FileChunk] | None,
|
|
) -> list[FileChunk]:
|
|
"""Attach cached embeddings to chunks whose hash already exists.
|
|
|
|
Returns the dirty subset (chunks still needing embeddings). Mutates
|
|
the input chunks in place: matched chunks get their `embedding`
|
|
field set from the cache.
|
|
"""
|
|
if not existing_chunks:
|
|
return list(chunks)
|
|
cached = {c.hash: c.embedding for c in existing_chunks if c.embedding}
|
|
if not cached:
|
|
return list(chunks)
|
|
dirty: list[FileChunk] = []
|
|
for c in chunks:
|
|
cached_emb = cached.get(c.hash)
|
|
if cached_emb is not None:
|
|
c.embedding = cached_emb
|
|
else:
|
|
dirty.append(c)
|
|
return dirty
|
|
|
|
@abstractmethod
|
|
async def parse(
|
|
self,
|
|
path: str,
|
|
existing_chunks: list[FileChunk] | None = None,
|
|
) -> ParsedFile:
|
|
"""Parse + embed the file, returning a fully populated `ParsedFile`.
|
|
|
|
`existing_chunks`, when provided, lets the parser reuse cached
|
|
embeddings for blocks whose hash hasn't changed.
|
|
"""
|