mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
51ec09d98f
- Replace legacy line-based chunking with AST tree approach that builds a complete document skeleton with content inlined under relevant sections - Add new chunking parameters: chunk_chars (default 2000) and embed_toc (default True) to control content size and TOC inclusion - Implement recursive chunking algorithm that respects structural boundaries (code lines, table rows, list items) and prevents splits inside blocks - Introduce part markers [Part X/N] for oversized leaf blocks that require splitting - Add CLI tool for inspecting parsed chunks and edges with options for preview and configuration - Refactor edge extraction to use FileEdge.from_text instead of parse_wikilinks for consistency BREAKING CHANGE: Chunk format changes significantly with full TOC skeleton wrapping content, affecting embedding models expecting breadcrumb prefixes.
29 lines
697 B
Python
29 lines
697 B
Python
"""Schema"""
|
|
|
|
from .application_config import ApplicationConfig, ComponentConfig, JobConfig
|
|
from .as_msg_stat import AsBlockStat, AsMsgStat
|
|
from .emb_node import EmbNode
|
|
from .chunk_filter import ChunkFilter
|
|
from .file_chunk import FileChunk
|
|
from .file_edge import FileEdge, extract_wikilinks
|
|
from .file_node import FileNode
|
|
from .request import Request
|
|
from .response import Response
|
|
from .stream_chunk import StreamChunk
|
|
|
|
__all__ = [
|
|
"ApplicationConfig",
|
|
"ComponentConfig",
|
|
"JobConfig",
|
|
"AsBlockStat",
|
|
"AsMsgStat",
|
|
"EmbNode",
|
|
"ChunkFilter",
|
|
"FileChunk",
|
|
"FileEdge",
|
|
"FileNode",
|
|
"Request",
|
|
"Response",
|
|
"StreamChunk",
|
|
"extract_wikilinks",
|
|
]
|