From 0adb7f231c82dbb96c0dd4856ddd00783d74d741 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Mon, 11 May 2026 19:35:01 +0800 Subject: [PATCH] up --- reme2/component/file_parser/__init__.py | 6 +- ...d_file_parser.py => linked_file_parser.py} | 2 +- .../component/file_parser/text_file_parser.py | 117 ---------- reme2/schema/__init__.py | 3 +- reme2/schema/file_node.py | 15 +- reme2/schema/parsed_file.py | 2 +- test/reme2/test_default_file_parser.py | 209 ++++++++++++++++++ 7 files changed, 225 insertions(+), 129 deletions(-) rename reme2/component/file_parser/{md_file_parser.py => linked_file_parser.py} (99%) delete mode 100644 reme2/component/file_parser/text_file_parser.py create mode 100644 test/reme2/test_default_file_parser.py diff --git a/reme2/component/file_parser/__init__.py b/reme2/component/file_parser/__init__.py index c10db242..0a2c8b86 100644 --- a/reme2/component/file_parser/__init__.py +++ b/reme2/component/file_parser/__init__.py @@ -2,12 +2,10 @@ from .base_file_parser import BaseFileParser from .default_file_parser import DefaultFileParser -from .md_file_parser import MdFileParser -from .text_file_parser import TextFileParser +from .linked_file_parser import LinkedFileParser __all__ = [ "BaseFileParser", "DefaultFileParser", - "MdFileParser", - "TextFileParser", + "LinkedFileParser", ] diff --git a/reme2/component/file_parser/md_file_parser.py b/reme2/component/file_parser/linked_file_parser.py similarity index 99% rename from reme2/component/file_parser/md_file_parser.py rename to reme2/component/file_parser/linked_file_parser.py index 7300e7b1..6a79dc5e 100644 --- a/reme2/component/file_parser/md_file_parser.py +++ b/reme2/component/file_parser/linked_file_parser.py @@ -36,7 +36,7 @@ from ...utils import hash_text @R.register("md") -class MdFileParser(BaseFileParser): +class LinkedFileParser(BaseFileParser): """Parser for Markdown files with YAML frontmatter and wikilink support.""" suffixes = [FileSuffixEnum.MD, FileSuffixEnum.MARKDOWN] diff --git a/reme2/component/file_parser/text_file_parser.py b/reme2/component/file_parser/text_file_parser.py deleted file mode 100644 index 193cb38a..00000000 --- a/reme2/component/file_parser/text_file_parser.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Text file parser with built-in chunking.""" - -import hashlib -from pathlib import Path - -import aiofiles - -from .base_file_parser import BaseFileParser -from ..component_registry import R -from ...enumeration import FileSuffixEnum -from ...schema import FileChunk, ParsedFile - - -@R.register("text") -class TextFileParser(BaseFileParser): - """Parser for text files with built-in chunking support.""" - - suffixes = [FileSuffixEnum.TXT] - - def __init__(self, encoding: str = "utf-8", chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs): - super().__init__(**kwargs) - self.encoding = encoding - self.chunk_size = max(32, chunk_tokens * 4) - self.overlap_size = max(0, chunk_overlap * 4) - - async def parse( - self, - path: str, - existing_chunks: list[FileChunk] | None = None, - ) -> ParsedFile: - file_path = Path(path) - stat = file_path.stat() - absolute_path = str(file_path.absolute()) - - try: - async with aiofiles.open(file_path, encoding=self.encoding) as f: - content = await f.read() - except UnicodeDecodeError: - async with aiofiles.open(file_path, encoding=self.encoding, errors="ignore") as f: - content = await f.read() - except Exception: - content = "" - - chunks = self._chunk(content, absolute_path) if content else [] - dirty = self._hash_diff_attach(chunks, existing_chunks) - if dirty: - await self._embed_chunks(dirty) - - return ParsedFile( - file=file_path.stem, - path=absolute_path, - st_mtime=stat.st_mtime, - chunks=chunks, - ) - - def _chunk(self, text: str, path: str) -> list[FileChunk]: - """Split text into chunks with overlap.""" - if not text.strip(): - return [] - - lines = text.split("\n") - chunks: list[FileChunk] = [] - buf: list[tuple[str, int]] = [] # (line_content, line_no) - buf_chars = 0 - - for line_no, line in enumerate(lines, 1): - # Split long lines into segments - for start in range(0, max(1, len(line)), self.chunk_size): - seg = line[start : start + self.chunk_size] - seg_chars = len(seg) + 1 # +1 for newline - - # Flush when buffer would exceed limit - if buf and buf_chars + seg_chars > self.chunk_size: - self._flush_chunk(chunks, buf, path) - buf, buf_chars = self._carry_overlap(buf) - - buf.append((seg, line_no)) - buf_chars += seg_chars - - if buf: - self._flush_chunk(chunks, buf, path) - - return [c for c in chunks if c.text.strip()] - - @staticmethod - def _flush_chunk(chunks: list[FileChunk], buf: list[tuple[str, int]], path: str): - """Create a chunk from buffer and append to chunks list.""" - chunk_text = "\n".join(content for content, _ in buf) - start_line, end_line = buf[0][1], buf[-1][1] - h = hashlib.sha256(chunk_text.encode()).hexdigest() - chunk_id = hashlib.sha256(f"{path}:{start_line}:{end_line}:{h}:{len(chunks)}".encode()).hexdigest() - - chunks.append( - FileChunk( - id=chunk_id, - path=path, - start_line=start_line, - end_line=end_line, - text=chunk_text, - hash=h, - ), - ) - - def _carry_overlap(self, buf: list[tuple[str, int]]) -> tuple[list[tuple[str, int]], int]: - """Keep overlapping lines from the end of buffer.""" - if self.overlap_size <= 0 or not buf: - return [], 0 - - acc, kept = 0, [] - for content, line_no in reversed(buf): - acc += len(content) + 1 - kept.insert(0, (content, line_no)) - if acc >= self.overlap_size: - break - - buf_chars = sum(len(c) + 1 for c, _ in kept) - return kept, buf_chars diff --git a/reme2/schema/__init__.py b/reme2/schema/__init__.py index cee3737b..f2f0b50b 100644 --- a/reme2/schema/__init__.py +++ b/reme2/schema/__init__.py @@ -6,7 +6,7 @@ from .emb_node import EmbNode from .chunk_filter import ChunkFilter from .file_chunk import FileChunk from .file_edge import FileEdge -from .file_node import FileNode +from .file_node import FileNode, FileMetadata from .parsed_file import ParsedFile from .request import Request from .response import Response @@ -23,6 +23,7 @@ __all__ = [ "FileChunk", "FileEdge", "FileNode", + "FileMetadata", "ParsedFile", "Request", "Response", diff --git a/reme2/schema/file_node.py b/reme2/schema/file_node.py index 303648e8..5105311e 100644 --- a/reme2/schema/file_node.py +++ b/reme2/schema/file_node.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, Field, ConfigDict +from __future__ import annotations -from reme.core.schema import FileMetadata +from pydantic import BaseModel, Field, ConfigDict class FileNode(BaseModel): @@ -8,21 +8,26 @@ class FileNode(BaseModel): path: str = Field(...) st_mtime: float = Field(...) - edges: list[FileEdge] = Field(default_factory=list) + edges: list["FileEdge"] = Field(default_factory=list) @property def file(self): ... - class FileMetadata(BaseModel): # FileNode -> FileMetadata path: str = Field(...) - edges: list[FileEdge] = Field(default_factory=list) + edges: list["FileEdge"] = Field(default_factory=list) title: str = Field(default="") description: str = Field(default="") tags: list[str] = Field(default_factory=list) + +# Rebuild models after FileEdge is defined +from .file_edge import FileEdge + +FileNode.model_rebuild() +FileMetadata.model_rebuild() \ No newline at end of file diff --git a/reme2/schema/parsed_file.py b/reme2/schema/parsed_file.py index 8e583cec..7fc81238 100644 --- a/reme2/schema/parsed_file.py +++ b/reme2/schema/parsed_file.py @@ -20,7 +20,7 @@ from pydantic import Field from .file_chunk import FileChunk from .file_edge import FileEdge -from .file_metadata import FileMetadata +from .file_node import FileMetadata class ParsedFile(FileMetadata): diff --git a/test/reme2/test_default_file_parser.py b/test/reme2/test_default_file_parser.py new file mode 100644 index 00000000..86905589 --- /dev/null +++ b/test/reme2/test_default_file_parser.py @@ -0,0 +1,209 @@ +"""Tests for DefaultFileParser.""" + +import asyncio +import tempfile +import os +from pathlib import Path + +# Add parent path for import +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from reme2.component.file_parser import DefaultFileParser + + +def test_parse_empty_file(): + """Test parsing an empty file.""" + + async def run(): + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + temp_path = f.name + + try: + parser = DefaultFileParser() + file_node, chunks = await parser.parse(temp_path) + assert file_node.path == temp_path + assert len(chunks) == 0 + print("✓ test_parse_empty_file passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_parse_small_file(): + """Test parsing a file smaller than chunk size.""" + + async def run(): + content = "Hello World\nThis is a test\nLine 3" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + f.write(content) + temp_path = f.name + + try: + parser = DefaultFileParser(chunk_byte_size=10000) + file_node, chunks = await parser.parse(temp_path) + assert len(chunks) == 1 + assert chunks[0].start_line == 1 + assert chunks[0].end_line == 3 + assert chunks[0].text == content + print("✓ test_parse_small_file passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_parse_multiline_file(): + """Test parsing a file with multiple lines.""" + + async def run(): + lines = ["Line 1", "Line 2", "Line 3", "Line 4", "Line 5"] + content = "\n".join(lines) + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + f.write(content) + temp_path = f.name + + try: + parser = DefaultFileParser(chunk_byte_size=10000) + file_node, chunks = await parser.parse(temp_path) + assert len(chunks) == 1 + assert chunks[0].start_line == 1 + assert chunks[0].end_line == 5 + print("✓ test_parse_multiline_file passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_parse_chunked_file(): + """Test parsing a file that requires multiple chunks.""" + + async def run(): + # Create content larger than chunk size + lines = ["A" * 100 for _ in range(200)] # ~20200 bytes + content = "\n".join(lines) + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + f.write(content) + temp_path = f.name + + try: + parser = DefaultFileParser(chunk_byte_size=5000, overlap_byte_size=100) + file_node, chunks = await parser.parse(temp_path) + assert len(chunks) > 1, f"Expected multiple chunks, got {len(chunks)}" + # Verify overlap by checking that consecutive chunks share some content + print(f" Created {len(chunks)} chunks") + print("✓ test_parse_chunked_file passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_parse_with_custom_encoding(): + """Test parsing a file with different encodings.""" + + async def run(): + content = "你好世界\n测试内容" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt", encoding="utf-8") as f: + f.write(content) + temp_path = f.name + + try: + parser = DefaultFileParser(encoding="utf-8") + file_node, chunks = await parser.parse(temp_path) + assert len(chunks) >= 1 + assert "你好世界" in chunks[0].text + print("✓ test_parse_with_custom_encoding passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_file_node_properties(): + """Test FileNode has correct properties.""" + + async def run(): + content = "test content" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + f.write(content) + temp_path = f.name + + try: + parser = DefaultFileParser() + file_node, chunks = await parser.parse(temp_path) + assert hasattr(file_node, "path") + assert hasattr(file_node, "st_mtime") + assert file_node.st_mtime > 0 + print("✓ test_file_node_properties passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_file_chunk_properties(): + """Test FileChunk has correct properties.""" + + async def run(): + content = "test content for chunk" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + f.write(content) + temp_path = f.name + + try: + parser = DefaultFileParser() + file_node, chunks = await parser.parse(temp_path) + chunk = chunks[0] + assert hasattr(chunk, "path") + assert hasattr(chunk, "start_line") + assert hasattr(chunk, "end_line") + assert hasattr(chunk, "text") + assert hasattr(chunk, "id") + assert hasattr(chunk, "hash") + assert chunk.start_line >= 1 + assert chunk.end_line >= chunk.start_line + print("✓ test_file_chunk_properties passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_min_chunk_and_overlap_size(): + """Test that minimum chunk and overlap sizes are enforced.""" + + async def run(): + # These values should be clamped to minimums + parser = DefaultFileParser(chunk_byte_size=1, overlap_byte_size=0) + assert parser.chunk_byte_size == 100 # minimum + assert parser.overlap_byte_size == 4 # minimum + + content = "test" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: + f.write(content) + temp_path = f.name + + try: + file_node, chunks = await parser.parse(temp_path) + assert len(chunks) == 1 + print("✓ test_min_chunk_and_overlap_size passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +if __name__ == "__main__": + test_parse_empty_file() + test_parse_small_file() + test_parse_multiline_file() + test_parse_chunked_file() + test_parse_with_custom_encoding() + test_file_node_properties() + test_file_chunk_properties() + test_min_chunk_and_overlap_size() + print("\n所有测试通过!") \ No newline at end of file