mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
514bf35050
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled
docs: add ReMe2 architecture design documentation - Add comprehensive design document (reme2.md) detailing the three-layer architecture (L1/L2/L3) for the vault system - Document new protocols for folder notes and memory management - Specify interface contracts for memory_* and vault_* tools - Outline implementation phases from current state to target refactor: fix typo in personal retriever class - Correct spelling error: 'retri eved_nodes' -> 'retrieved_nodes' in PersonalRetriever.result assignment chore: update gitignore with vault-related patterns - Add '/vault' to ignore vault directory - Add '/reme-plugin' to ignore plugin files - Add '/reme2/vault' to ignore new vault implementation ```
117 lines
3.9 KiB
Python
117 lines
3.9 KiB
Python
"""Text file parser with built-in chunking."""
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
import aiofiles
|
|
|
|
from .base_file_parser import BaseFileParser
|
|
from ..component_registry import R
|
|
from ...enumeration import FileSuffixEnum
|
|
from ...schema import FileChunk, ParsedFile
|
|
|
|
|
|
@R.register("text")
|
|
class TextFileParser(BaseFileParser):
|
|
"""Parser for text files with built-in chunking support."""
|
|
|
|
suffixes = [FileSuffixEnum.TXT]
|
|
|
|
def __init__(self, encoding: str = "utf-8", chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.encoding = encoding
|
|
self.chunk_size = max(32, chunk_tokens * 4)
|
|
self.overlap_size = max(0, chunk_overlap * 4)
|
|
|
|
async def parse(
|
|
self,
|
|
path: str,
|
|
existing_chunks: list[FileChunk] | None = None,
|
|
) -> ParsedFile:
|
|
file_path = Path(path)
|
|
stat = file_path.stat()
|
|
absolute_path = str(file_path.absolute())
|
|
|
|
try:
|
|
async with aiofiles.open(file_path, encoding=self.encoding) as f:
|
|
content = await f.read()
|
|
except UnicodeDecodeError:
|
|
async with aiofiles.open(file_path, encoding=self.encoding, errors="ignore") as f:
|
|
content = await f.read()
|
|
except Exception:
|
|
content = ""
|
|
|
|
chunks = self._chunk(content, absolute_path) if content else []
|
|
dirty = self._hash_diff_attach(chunks, existing_chunks)
|
|
if dirty:
|
|
await self._embed_chunks(dirty)
|
|
|
|
return ParsedFile(
|
|
file=file_path.stem,
|
|
path=absolute_path,
|
|
st_mtime=stat.st_mtime,
|
|
chunks=chunks,
|
|
)
|
|
|
|
def _chunk(self, text: str, path: str) -> list[FileChunk]:
|
|
"""Split text into chunks with overlap."""
|
|
if not text.strip():
|
|
return []
|
|
|
|
lines = text.split("\n")
|
|
chunks: list[FileChunk] = []
|
|
buf: list[tuple[str, int]] = [] # (line_content, line_no)
|
|
buf_chars = 0
|
|
|
|
for line_no, line in enumerate(lines, 1):
|
|
# Split long lines into segments
|
|
for start in range(0, max(1, len(line)), self.chunk_size):
|
|
seg = line[start : start + self.chunk_size]
|
|
seg_chars = len(seg) + 1 # +1 for newline
|
|
|
|
# Flush when buffer would exceed limit
|
|
if buf and buf_chars + seg_chars > self.chunk_size:
|
|
self._flush_chunk(chunks, buf, path)
|
|
buf, buf_chars = self._carry_overlap(buf)
|
|
|
|
buf.append((seg, line_no))
|
|
buf_chars += seg_chars
|
|
|
|
if buf:
|
|
self._flush_chunk(chunks, buf, path)
|
|
|
|
return [c for c in chunks if c.text.strip()]
|
|
|
|
@staticmethod
|
|
def _flush_chunk(chunks: list[FileChunk], buf: list[tuple[str, int]], path: str):
|
|
"""Create a chunk from buffer and append to chunks list."""
|
|
chunk_text = "\n".join(content for content, _ in buf)
|
|
start_line, end_line = buf[0][1], buf[-1][1]
|
|
h = hashlib.sha256(chunk_text.encode()).hexdigest()
|
|
chunk_id = hashlib.sha256(f"{path}:{start_line}:{end_line}:{h}:{len(chunks)}".encode()).hexdigest()
|
|
|
|
chunks.append(
|
|
FileChunk(
|
|
id=chunk_id,
|
|
path=path,
|
|
start_line=start_line,
|
|
end_line=end_line,
|
|
text=chunk_text,
|
|
hash=h,
|
|
),
|
|
)
|
|
|
|
def _carry_overlap(self, buf: list[tuple[str, int]]) -> tuple[list[tuple[str, int]], int]:
|
|
"""Keep overlapping lines from the end of buffer."""
|
|
if self.overlap_size <= 0 or not buf:
|
|
return [], 0
|
|
|
|
acc, kept = 0, []
|
|
for content, line_no in reversed(buf):
|
|
acc += len(content) + 1
|
|
kept.insert(0, (content, line_no))
|
|
if acc >= self.overlap_size:
|
|
break
|
|
|
|
buf_chars = sum(len(c) + 1 for c, _ in kept)
|
|
return kept, buf_chars
|