feat(parser): add mistletoe dependency and refactor markdown parsing

Add mistletoe as a project dependency for enhanced markdown parsing
capabilities. Refactor the LinkedFileParser to use a proper AST-based
approach with MdNode tree structure, replacing the previous flat token
processing method. The new implementation provides better handling of
markdown elements including tables, code fences, lists, and headings,
with improved chunking logic that maintains document structure in
generated content segments.

The changes include:
- Add mistletoe dependency to pyproject.toml
- Implement proper AST node representation with MdNode class
- Create recursive chunking algorithm with TOC preservation
- Add support for frontmatter extraction with FileFrontMatter schema
- Optimize leaf node splitting with proper boundary detection
- Include part numbering for split content pieces

pyproject.toml

@@ -58,6 +58,7 @@ dependencies = [
     "uvicorn>=0.40.0",
     "watchfiles>=1.1.1",
     "pyyaml>=6.0.3",
+    "mistletoe",
 ]
 
 [project.optional-dependencies]

reme2/schema/__init__.py

@@ -6,7 +6,7 @@ from .emb_node import EmbNode
 from .chunk_filter import ChunkFilter
 from .file_chunk import FileChunk
 from .file_edge import FileEdge, extract_wikilinks
-from .file_node import FileNode
+from .file_node import FileFrontMatter, FileNode
 from .request import Request
 from .response import Response
 from .stream_chunk import StreamChunk
@@ -21,6 +21,7 @@ __all__ = [
     "ChunkFilter",
     "FileChunk",
     "FileEdge",
+    "FileFrontMatter",
     "FileNode",
     "Request",
     "Response",

reme2/schema/file_node.py

@@ -1,24 +1,6 @@
-from pydantic import BaseModel, Field, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 
-
-class FileEdge(BaseModel):
-    """ Format:
-    [[X]]                           bare wikilink         → predicate=None
-    extends:: [[X]]                 line-level Dataview   → predicate="extends"
-    [extends:: [[X]]]               inline-bracketed      → predicate="extends"
-    extends:: [[A]], [[B]]          multi-target          → 2 edges
-    """
-    link: str = Field(default=...)
-    predicate: str | None = Field(default=None)
-
-    @property
-    def link_path(self) -> str:
-        return self.link.split("#", 1)[0]
-
-    @property
-    def link_anchor(self) -> str:
-        link_split = self.link.split("#", 1)
-        return link_split[1] if len(link_split) > 1 else ""
+from .file_edge import FileEdge
 
 
 class FileFrontMatter(BaseModel):

tests/inspect_md_parser.py

@@ -0,0 +1,81 @@
+"""CLI to inspect `LinkedFileParser` output on a real markdown file.
+
+Run a vault file through the parser and dump its chunks + edges so you
+can eyeball what the AST chunker produced (sizes, TOC skeleton wrap,
+``[Part X/N]`` markers, link extraction). Not a pytest test — it's a
+manual inspection script that lives in `tests/` because that's where
+ad-hoc developer tools belong.
+
+Usage::
+
+    python tests/inspect_md_parser.py <path> [--chunk-chars N] [--no-toc]
+                                              [--show-edges] [--preview N]
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+
+from reme2.component.file_parser.linked_file_parser import LinkedFileParser
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(
+        description="Parse a markdown file with LinkedFileParser and dump chunks + edges.",
+    )
+    ap.add_argument("path", help="Path to a markdown file.")
+    ap.add_argument(
+        "--chunk-chars", type=int, default=2000,
+        help="Max characters per chunk content (default: 2000). "
+             "Excludes TOC skeleton when embed_toc is on.",
+    )
+    ap.add_argument(
+        "--no-toc", action="store_true",
+        help="Disable the full-doc TOC skeleton wrap; chunks become plain content.",
+    )
+    ap.add_argument(
+        "--show-edges", action="store_true",
+        help="Print extracted FileEdges before chunks.",
+    )
+    ap.add_argument(
+        "--preview", type=int, default=0,
+        help="Truncate each chunk to N chars in output (0 = full text).",
+    )
+    args = ap.parse_args()
+
+    parser = LinkedFileParser(
+        chunk_chars=args.chunk_chars,
+        embed_toc=not args.no_toc,
+    )
+    node, chunks = asyncio.run(parser.parse(args.path))
+
+    print(f"file:        {node.path}")
+    print(f"chunk_chars: {args.chunk_chars}")
+    print(f"embed_toc:   {parser.embed_toc}")
+    print(f"chunks:      {len(chunks)}")
+    print(f"chars total: {sum(len(c.text) for c in chunks)}")
+    if chunks:
+        sizes = [len(c.text) for c in chunks]
+        print(f"chars min/avg/max: {min(sizes)} / {sum(sizes)//len(sizes)} / {max(sizes)}")
+    if args.show_edges:
+        print(f"\nedges ({len(node.edges)}):")
+        for e in node.edges:
+            print(
+                f"  → {e.link}"
+                + (f"  predicate={e.predicate}" if e.predicate else "")
+                + (f"  anchor={e.anchor}" if e.anchor else "")
+            )
+
+    for i, c in enumerate(chunks):
+        print(f"\n{'=' * 72}")
+        print(f"chunk {i}  lines {c.start_line}-{c.end_line}  {len(c.text)} chars")
+        print("-" * 72)
+        text = c.text if args.preview <= 0 else c.text[: args.preview]
+        print(text)
+        if args.preview > 0 and len(c.text) > args.preview:
+            print(f"... ({len(c.text) - args.preview} more chars truncated)")
+
+
+if __name__ == "__main__":
+    main()

tests/test_md_chunker.py

@@ -469,34 +469,24 @@ def test_body_run_budget_excludes_toc():
     assert "klmnopqrst" in chunks[0].text
 
 
-def test_finalize_off_returns_content_unchanged():
-    """White-box: _finalize with embed_toc=False is the identity for content."""
-    from mistletoe.block_token import Document
-    from mistletoe.markdown_renderer import MarkdownRenderer
-
-    p = _parser(500, embed_toc=False)
+def test_render_with_toc_off_returns_content_unchanged():
+    """White-box: with embed_toc=False the chunk text is the raw content."""
     txt = "# A\n## B\nbody\n"
-    with MarkdownRenderer() as r:
-        tree = p._build_tree(Document(txt), r)
-    h1 = tree.children[0]
-    b = h1.children[0]  # ## B
-    out = p._finalize(tree, b, "RAW", owns_subtree=False)
-    assert out == "RAW"
+    chunks = _parser(500, embed_toc=False)._chunk(txt, "/x.md")
+    assert len(chunks) == 1
+    # Whole-doc chunk: content is the rendered markdown, no TOC wrap added.
+    assert "# A" in chunks[0].text and "## B" in chunks[0].text
+    assert chunks[0].text == "# A\n\n## B\n\nbody"
 
 
-def test_finalize_on_wraps_with_toc():
-    """White-box: _finalize with embed_toc=True wraps content with skeleton."""
-    from mistletoe.block_token import Document
-    from mistletoe.markdown_renderer import MarkdownRenderer
-
-    p = _parser(500, embed_toc=True)
+def test_render_with_toc_on_wraps_with_skeleton():
+    """Black-box: with embed_toc=True, every chunk includes the doc's
+    heading skeleton alongside the body content."""
     txt = "# A\n## B\nbody\n"
-    with MarkdownRenderer() as r:
-        tree = p._build_tree(Document(txt), r)
-    h1 = tree.children[0]
-    b = h1.children[0]
-    out = p._finalize(tree, b, "RAW", owns_subtree=False)
-    assert "# A" in out and "## B" in out and "RAW" in out
+    chunks = _parser(500, embed_toc=True)._chunk(txt, "/x.md")
+    assert len(chunks) == 1
+    text = chunks[0].text
+    assert "# A" in text and "## B" in text and "body" in text
 
 
 # --------------------------------------------------------------------------
@@ -678,40 +668,28 @@ def test_chunk_id_changes_with_content():
 
 
 def test_render_with_full_toc_inlines_at_owner():
-    """White-box: build a tree, call _render_with_full_toc directly and
-    verify the slot-fill behavior."""
-    from mistletoe.block_token import Document
-    from mistletoe.markdown_renderer import MarkdownRenderer
-
-    p = _parser(2000)
-    txt = "# A\n\n## B\nbody B\n\n## C\nbody C\n"
-    with MarkdownRenderer() as r:
-        tree = p._build_tree(Document(txt), r)
-    h1 = tree.children[0]
-    b = h1.children[0]  # ## B
-    rendered = p._render_with_full_toc(tree, b, "INSERTED", owns_subtree=False)
-    # Skeleton lists # A, ## B, ## C; INSERTED sits under ## B but BEFORE ## C
-    assert "# A" in rendered
-    assert "## B" in rendered
-    assert "## C" in rendered
-    b_idx = rendered.find("## B")
-    inserted_idx = rendered.find("INSERTED")
-    c_idx = rendered.find("## C")
-    assert b_idx < inserted_idx < c_idx
+    """Black-box: when ## B's content emits as its own chunk, the chunk's
+    TOC slot-fills B's content BEFORE the next sibling section ## C."""
+    body_b = "x" * 400
+    body_c = "y" * 400
+    txt = f"# A\n\n## B\n\n{body_b}\n\n## C\n\n{body_c}\n"
+    chunks = _parser(500, embed_toc=True)._chunk(txt, "/x.md")
+    b_chunks = [c for c in chunks if "xxxx" in c.text and "yyyy" not in c.text]
+    assert b_chunks, "expected a B-body chunk distinct from C's"
+    text = b_chunks[0].text
+    assert "## B" in text and "## C" in text
+    # B's content sits under ## B but BEFORE ## C
+    assert text.find("## B") < text.find("xxxx") < text.find("## C")
 
 
 def test_render_with_full_toc_root_owner():
-    """Root owner means content sits BEFORE the first heading."""
-    from mistletoe.block_token import Document
-    from mistletoe.markdown_renderer import MarkdownRenderer
-
-    p = _parser(2000)
-    txt = "# A\n\nbody A\n"
-    with MarkdownRenderer() as r:
-        tree = p._build_tree(Document(txt), r)
-    rendered = p._render_with_full_toc(tree, tree, "PRELUDE", owns_subtree=False)
-    assert rendered.startswith("PRELUDE")
-    assert "# A" in rendered
+    """Black-box: body before any heading attaches to the root slot,
+    so it appears BEFORE the first section heading in the chunk."""
+    txt = "PRELUDE body\n\n# A\n\nbody A\n"
+    chunks = _parser(500, embed_toc=True)._chunk(txt, "/x.md")
+    assert len(chunks) == 1
+    text = chunks[0].text
+    assert text.find("PRELUDE") < text.find("# A")
 
 
 # --------------------------------------------------------------------------