mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
6a5561f86a
- Add file_graph import to component registry - Register FILE_GRAPH enum in ComponentEnum - Implement BaseFileGraph integration in LinkedFileParser - Replace FileEdge with FileLink for better semantic clarity - Add lazy resolution of file_graph from app_context - Update file watcher logging to reflect links instead of edges refactor: streamline MCP transport layer architecture - Remove redundant step shells from reme2/mcp/steps/ - Consolidate all @R.register components to reme2.memory package - Update server.py to import reme2.memory directly - Revise README.md to document new architecture - Simplify module dependencies and import structure
86 lines
2.8 KiB
Python
86 lines
2.8 KiB
Python
"""CLI to inspect `LinkedFileParser` output on a real markdown file.
|
|
|
|
Run a vault file through the parser and dump its chunks + edges so you
|
|
can eyeball what the AST chunker produced (sizes, TOC skeleton wrap,
|
|
``[Part X/N]`` markers, link extraction). Not a pytest test — it's a
|
|
manual inspection script that lives in `tests/` because that's where
|
|
ad-hoc developer tools belong.
|
|
|
|
Usage::
|
|
|
|
python tests/inspect_md_parser.py <path> [--chunk-chars N] [--no-toc]
|
|
[--show-edges] [--preview N]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
|
|
from reme2.component.file_parser.linked_file_parser import LinkedFileParser
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(
|
|
description="Parse a markdown file with LinkedFileParser and dump chunks + edges.",
|
|
)
|
|
ap.add_argument("path", help="Path to a markdown file.")
|
|
ap.add_argument(
|
|
"--chunk-chars",
|
|
type=int,
|
|
default=2000,
|
|
help="Max characters per chunk content (default: 2000). " "Excludes TOC skeleton when embed_toc is on.",
|
|
)
|
|
ap.add_argument(
|
|
"--no-toc",
|
|
action="store_true",
|
|
help="Disable the full-doc TOC skeleton wrap; chunks become plain content.",
|
|
)
|
|
ap.add_argument(
|
|
"--show-edges",
|
|
action="store_true",
|
|
help="Print extracted FileLinks before chunks.",
|
|
)
|
|
ap.add_argument(
|
|
"--preview",
|
|
type=int,
|
|
default=0,
|
|
help="Truncate each chunk to N chars in output (0 = full text).",
|
|
)
|
|
args = ap.parse_args()
|
|
|
|
parser = LinkedFileParser(
|
|
chunk_chars=args.chunk_chars,
|
|
embed_toc=not args.no_toc,
|
|
)
|
|
node, chunks = asyncio.run(parser.parse(args.path))
|
|
|
|
print(f"file: {node.path}")
|
|
print(f"chunk_chars: {args.chunk_chars}")
|
|
print(f"embed_toc: {parser.embed_toc}")
|
|
print(f"chunks: {len(chunks)}")
|
|
print(f"chars total: {sum(len(c.text) for c in chunks)}")
|
|
if chunks:
|
|
sizes = [len(c.text) for c in chunks]
|
|
print(f"chars min/avg/max: {min(sizes)} / {sum(sizes)//len(sizes)} / {max(sizes)}")
|
|
if args.show_edges:
|
|
print(f"\nlinks ({len(node.links)}):")
|
|
for link in node.links:
|
|
print(
|
|
f" → {link.path}"
|
|
+ (f" predicate={link.predicate}" if link.predicate else "")
|
|
+ (f" anchor={link.anchor}" if link.anchor else ""),
|
|
)
|
|
|
|
for i, c in enumerate(chunks):
|
|
print(f"\n{'=' * 72}")
|
|
print(f"chunk {i} lines {c.start_line}-{c.end_line} {len(c.text)} chars")
|
|
print("-" * 72)
|
|
text = c.text if args.preview <= 0 else c.text[: args.preview]
|
|
print(text)
|
|
if args.preview > 0 and len(c.text) > args.preview:
|
|
print(f"... ({len(c.text) - args.preview} more chars truncated)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|