mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
37f0037e6b
Add mistletoe as a project dependency for enhanced markdown parsing capabilities. Refactor the LinkedFileParser to use a proper AST-based approach with MdNode tree structure, replacing the previous flat token processing method. The new implementation provides better handling of markdown elements including tables, code fences, lists, and headings, with improved chunking logic that maintains document structure in generated content segments. The changes include: - Add mistletoe dependency to pyproject.toml - Implement proper AST node representation with MdNode class - Create recursive chunking algorithm with TOC preservation - Add support for frontmatter extraction with FileFrontMatter schema - Optimize leaf node splitting with proper boundary detection - Include part numbering for split content pieces
30 lines
737 B
Python
30 lines
737 B
Python
"""Schema"""
|
|
|
|
from .application_config import ApplicationConfig, ComponentConfig, JobConfig
|
|
from .as_msg_stat import AsBlockStat, AsMsgStat
|
|
from .emb_node import EmbNode
|
|
from .chunk_filter import ChunkFilter
|
|
from .file_chunk import FileChunk
|
|
from .file_edge import FileEdge, extract_wikilinks
|
|
from .file_node import FileFrontMatter, FileNode
|
|
from .request import Request
|
|
from .response import Response
|
|
from .stream_chunk import StreamChunk
|
|
|
|
__all__ = [
|
|
"ApplicationConfig",
|
|
"ComponentConfig",
|
|
"JobConfig",
|
|
"AsBlockStat",
|
|
"AsMsgStat",
|
|
"EmbNode",
|
|
"ChunkFilter",
|
|
"FileChunk",
|
|
"FileEdge",
|
|
"FileFrontMatter",
|
|
"FileNode",
|
|
"Request",
|
|
"Response",
|
|
"StreamChunk",
|
|
"extract_wikilinks",
|
|
]
|