mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
3fc3fd65e8
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled
- Introduce BaseFileParser abstract class with component registration - Add MdFileParser implementation for markdown files with YAML frontmatter - Create TextFileParser implementation with built-in chunking support - Implement file suffix enumeration for parser type safety - Add chunking logic with configurable token size and overlap - Support text file parsing with error handling for encoding issues - Include line number tracking and content hashing for file chunks
18 lines
575 B
Python
18 lines
575 B
Python
"""Abstract base class for file parsers."""
|
|
|
|
from abc import abstractmethod
|
|
|
|
from ..base_component import BaseComponent
|
|
from ...enumeration import ComponentEnum, FileSuffixEnum
|
|
from ...schema import FileChunk, FileMetadata
|
|
|
|
|
|
class BaseFileParser(BaseComponent):
|
|
"""Parser that declares handled suffixes and produces FileChunks."""
|
|
|
|
component_type = ComponentEnum.FILE_PARSER
|
|
suffixes: list[FileSuffixEnum] = []
|
|
|
|
@abstractmethod
|
|
async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]:
|
|
"""Parse a file into metadata and chunks."""
|