ReMe/tests/unit/test_markdown_file_chunker.py

"""Tests for MarkdownFileChunker (markdown parser + wikilink extraction).

Wikilink convention here is strict: targets are taken literally, no
short-form basename search, no implicit ``.md``, no folder-note
expansion. ``lint:dangling`` handles validation; the parser is just
a markdown-to-FileNode transformer.
"""

# pylint: disable=protected-access

from contextlib import chdir as temp_chdir
import asyncio
import os
import tempfile
from unittest.mock import patch

import pytest

from reme.components.file_chunker import DefaultFileChunker, MarkdownFileChunker


def _write_md(tmpdir: str, name: str, body: str) -> str:
    """Drop a markdown file under tmpdir, return its relative path (matches cwd)."""
    if "/" in name:
        os.makedirs(os.path.join(tmpdir, os.path.dirname(name)), exist_ok=True)
    with open(os.path.join(tmpdir, name), "w", encoding="utf-8") as f:
        f.write(body)
    return name


@pytest.mark.asyncio
async def test_chunk_keeps_pre_read_mtime_when_source_changes(tmp_path, monkeypatch):
    """A concurrent write must remain visible to the next filesystem scan."""
    monkeypatch.chdir(tmp_path)
    path = tmp_path / "changing.md"
    path.write_text("", encoding="utf-8")
    os.utime(path, (1000, 1000))
    chunker = MarkdownFileChunker()

    async def read_then_finish_write(_path):
        path.write_text("Completed memory note.", encoding="utf-8")
        os.utime(path, (2000, 2000))
        return ""

    monkeypatch.setattr(chunker, "_read_text_for_indexing", read_then_finish_write)
    node, chunks = await chunker.chunk("changing.md")

    assert chunks == []
    assert node.st_mtime == 1000
    assert path.stat().st_mtime != node.st_mtime


@pytest.mark.asyncio
async def test_parse_empty_file():
    """An empty .md → FileNode, no chunks, no links."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        path = _write_md(tmp, "x.md", "")
        chunker = MarkdownFileChunker()
        node, chunks = await chunker.chunk(path)
        assert node.path == "x.md"
        assert chunks == []
        assert node.links == []


@pytest.mark.asyncio
async def test_invalid_utf8_is_replaced_without_modifying_source():
    """Bad source bytes degrade the derived index but remain untouched on disk."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        source = b"# valid\ncontent before \xcd content after\n"
        path = os.path.join(tmp, "invalid.md")
        with open(path, "wb") as f:
            f.write(source)

        chunker = MarkdownFileChunker()
        with patch.object(chunker.logger, "warning") as warning:
            node, chunks = await chunker.chunk("invalid.md")

        assert node.path == "invalid.md"
        assert "content before \ufffd content after" in "\n".join(chunk.text for chunk in chunks)
        with open(path, "rb") as f:
            assert f.read() == source
        warning.assert_called_once_with(
            "Invalid utf-8 in invalid.md at byte 23 (bytes: cd); "
            "indexed with replacement characters; source file unchanged",
        )


@pytest.mark.asyncio
async def test_invalid_utf8_strict_policy_still_raises():
    """Strict mode remains available when callers require exact decoding."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        with open(os.path.join(tmp, "invalid.md"), "wb") as f:
            f.write(b"valid\xcdinvalid")

        chunker = MarkdownFileChunker(invalid_encoding_policy="strict")
        try:
            await chunker.chunk("invalid.md")
        except UnicodeDecodeError as exc:
            assert exc.start == 5
        else:
            raise AssertionError("strict policy must reject invalid UTF-8")


def test_constructor_preserves_positional_arguments():
    """The decoding policy must not shift the established positional parameters."""
    chunker = MarkdownFileChunker("utf-8", 5000, False, 10, True, ["name"])

    assert chunker.encoding == "utf-8"
    assert chunker.chunk_byte_size == 5000
    assert chunker.embed_toc is False
    assert chunker.max_ast_sections == 10
    assert chunker.include_frontmatter_in_metadata is True
    assert chunker.include_frontmatter_keys_in_metadata == ["name"]


@pytest.mark.asyncio
async def test_plain_text_fallback_normalizes_newlines():
    """Markdown fallback chunks stay stable for equivalent platform newlines."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        path = os.path.join(tmp, "fallback.md")
        with open(path, "wb") as f:
            f.write(b"# One\r\nbody\r# Two\r\nbody\r\n")

        chunker = MarkdownFileChunker(max_ast_sections=0)
        _, original_chunks = await chunker.chunk("fallback.md")
        with open(path, "wb") as f:
            f.write(b"# One\nbody\n# Two\nbody\n")
        _, normalized_chunks = await chunker.chunk("fallback.md")

        assert [chunk.text for chunk in original_chunks] == [chunk.text for chunk in normalized_chunks]
        assert [chunk.id for chunk in original_chunks] == [chunk.id for chunk in normalized_chunks]


@pytest.mark.asyncio
async def test_invalid_ascii_is_replaced_for_markdown():
    """Markdown byte accounting accepts the configured codec's replacement text."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        source = b"# valid\ncontent before \xff content after\n"
        path = os.path.join(tmp, "invalid.md")
        with open(path, "wb") as f:
            f.write(source)

        chunker = MarkdownFileChunker(encoding="ascii")
        _, chunks = await chunker.chunk("invalid.md")

        assert "content before ? content after" in "\n".join(chunk.text for chunk in chunks)
        with open(path, "rb") as f:
            assert f.read() == source


@pytest.mark.asyncio
async def test_parse_frontmatter_only():
    """A file with only frontmatter (no body) → no chunks, no links."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        path = _write_md(tmp, "fm.md", "---\nname: t\n---\n")
        chunker = MarkdownFileChunker()
        node, chunks = await chunker.chunk(path)
        assert node.front_matter.name == "t"
        assert chunks == []
        assert node.links == []


@pytest.mark.parametrize("max_ast_sections", [0, None])
def test_markdown_source_lines_and_metadata_remain_separate(tmp_path, max_ast_sections):
    """AST and inherited byte fallback retain Markdown-specific semantics."""
    path = tmp_path / "note.md"
    source = b"---\nname: t\n---\n# H\nline 1\n[[target.md]]\n"
    path.write_bytes(source)
    chunker = MarkdownFileChunker(max_ast_sections=max_ast_sections, include_frontmatter_in_metadata=True)

    node, chunks = asyncio.run(chunker.chunk(path))

    assert len(chunks) == 1
    assert (chunks[0].start_line, chunks[0].end_line) == (4, 6)
    assert "name: t" not in chunks[0].text
    assert chunks[0].metadata == {"name": "t"}
    assert node.front_matter.name == "t"
    assert [link.target_path for link in node.links] == ["target.md"]
    assert node.chunk_ids == [chunks[0].id]
    assert path.read_bytes() == source


@pytest.mark.asyncio
async def test_parse_frontmatter_metadata_is_opt_in():
    """Chunk metadata preserves the old empty default unless explicitly enabled."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = (
            "---\n"
            "name: locomo-event\n"
            "description: Jon lost his job\n"
            "conversation_date: 2023-01-19\n"
            "---\n"
            "Jon said he lost his job today.\n"
        )
        path = _write_md(tmp, "daily/2023-01-19/locomo-event.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=500)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) == 1
        assert chunks[0].metadata == {}

        chunker = MarkdownFileChunker(chunk_byte_size=500, include_frontmatter_in_metadata=True)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) == 1
        assert chunks[0].metadata == {
            "name": "locomo-event",
            "description": "Jon lost his job",
            "conversation_date": "2023-01-19",
        }


@pytest.mark.asyncio
async def test_parse_frontmatter_metadata_keys_allowlist():
    """``include_frontmatter_keys_in_metadata`` restricts copied keys to an allow-list."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = (
            "---\n"
            "name: locomo-event\n"
            "description: Jon lost his job\n"
            "conversation_date: 2023-01-19\n"
            "---\n"
            "Jon said he lost his job today.\n"
        )
        path = _write_md(tmp, "daily/2023-01-19/locomo-event.md", body)

        # Allow-list restricted to a single key.
        chunker = MarkdownFileChunker(
            chunk_byte_size=500,
            include_frontmatter_in_metadata=True,
            include_frontmatter_keys_in_metadata=["conversation_date"],
        )
        _, chunks = await chunker.chunk(path)
        assert len(chunks) == 1
        assert chunks[0].metadata == {"conversation_date": "2023-01-19"}

        # Allow-list with a key not present in frontmatter is a no-op for that key.
        chunker = MarkdownFileChunker(
            chunk_byte_size=500,
            include_frontmatter_in_metadata=True,
            include_frontmatter_keys_in_metadata=["conversation_date", "absent"],
        )
        _, chunks = await chunker.chunk(path)
        assert chunks[0].metadata == {"conversation_date": "2023-01-19"}

        # Empty allow-list (not None) keeps the legacy "all non-empty keys" behavior.
        chunker = MarkdownFileChunker(
            chunk_byte_size=500,
            include_frontmatter_in_metadata=True,
            include_frontmatter_keys_in_metadata=[],
        )
        _, chunks = await chunker.chunk(path)
        assert chunks[0].metadata == {
            "name": "locomo-event",
            "description": "Jon lost his job",
            "conversation_date": "2023-01-19",
        }

        # Allow-list is ignored when the master toggle is off (back-compat default).
        chunker = MarkdownFileChunker(
            chunk_byte_size=500,
            include_frontmatter_in_metadata=False,
            include_frontmatter_keys_in_metadata=["conversation_date"],
        )
        _, chunks = await chunker.chunk(path)
        assert chunks[0].metadata == {}


@pytest.mark.asyncio
async def test_parse_small_body_one_chunk():
    """A body shorter than chunk_byte_size produces exactly one chunk that contains the body."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "# Hello\n\nthis is a small body."
        path = _write_md(tmp, "small.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=500)
        node, chunks = await chunker.chunk(path)
        assert len(chunks) == 1
        assert "this is a small body" in chunks[0].text
        assert node.chunk_ids == [chunks[0].id]


@pytest.mark.asyncio
async def test_parse_small_children_are_cached_without_recursive_calls():
    """An oversized parent greedily caches fitting children without recursing into them."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        sections = "\n\n".join(f"# Section-{i}\n\n{'x' * 30}" for i in range(6))
        path = _write_md(tmp, "small-children.md", sections)
        chunker = MarkdownFileChunker(chunk_byte_size=100, embed_toc=False)
        with patch.object(
            chunker,
            "_chunk_node",
            wraps=chunker._chunk_node,
        ) as chunk_node:
            _, chunks = await chunker.chunk(path)

        assert chunk_node.call_count == 1
        assert 1 < len(chunks) < 6
        assert all(len(chunk.text.encode("utf-8")) <= chunker.chunk_byte_size for chunk in chunks)


@pytest.mark.asyncio
async def test_parse_child_cache_flushes_at_exact_limit():
    """A cache reaching ``chunk_byte_size`` is finalized before the next child."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        path = _write_md(tmp, "exact-cache.md", f"# A\n\n{'x' * 95}\n\n# B\n\ny")
        chunker = MarkdownFileChunker(chunk_byte_size=100)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) == 2
        assert len(chunks[0].text) == chunker.chunk_byte_size
        assert chunks[0].text.startswith("# A")
        assert chunks[1].text == "# B\n\ny"


@pytest.mark.asyncio
async def test_parse_oversized_child_flushes_parent_cache():
    """Recursive child chunks do not merge across the parent cache boundary."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        leaves = "\n\n".join(f"## L{i}\n\n{'x' * 10}" for i in range(6))
        body = f"# A\n\na\n\n# Large\n\n{leaves}\n\n# C\n\nc"
        path = _write_md(tmp, "recursive-boundary.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=100, embed_toc=False)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) == 4
        assert chunks[0].text == "# A\n\na"
        assert chunks[-2].text == f"## L4\n\n{'x' * 10}\n\n## L5\n\n{'x' * 10}"
        assert chunks[-1].text == "# C\n\nc"


@pytest.mark.asyncio
async def test_parse_oversized_body_splits():
    """A body exceeding chunk_byte_size triggers multiple chunks."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        paras = "\n\n".join(f"paragraph {i} with some content text here." for i in range(50))
        body = "# H\n\n" + paras
        path = _write_md(tmp, "big.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=200)
        node, chunks = await chunker.chunk(path)
        assert len(chunks) > 1
        assert all(len(chunk.text.encode("utf-8")) <= chunker.chunk_byte_size for chunk in chunks)
        assert node.chunk_ids == [chunk.id for chunk in chunks]


@pytest.mark.asyncio
async def test_parse_links_literal_targets():
    """Wikilink targets are taken verbatim — full path → FileLink.target_path."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "see [[topics/Alice.md]] and [[topics/Bob.md#sec]]"
        path = _write_md(tmp, "note.md", body)
        chunker = MarkdownFileChunker()
        node, _ = await chunker.chunk(path)
        pairs = {(link.target_path, link.target_anchor) for link in node.links}
        assert ("topics/Alice.md", None) in pairs
        assert ("topics/Bob.md", "sec") in pairs
        # source_path always equals the node's own path
        for link in node.links:
            assert link.source_path == node.path


@pytest.mark.asyncio
async def test_parse_links_short_and_no_ext_kept_literally():
    """Short and no-ext forms are NOT resolved — they're stored as-is.

    The parser does no resolution; whether the target exists is a
    ``lint:dangling`` concern. ``[[Alice]]`` becomes
    ``target_path='Alice'`` and will be flagged dangling unless a node
    with literal path 'Alice' actually exists.
    """

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "see [[Alice]] and [[topics/Alice]] but also [[topics/Alice.md]]"
        path = _write_md(tmp, "note.md", body)
        chunker = MarkdownFileChunker()
        node, _ = await chunker.chunk(path)
        targets = {link.target_path for link in node.links}
        assert targets == {"Alice", "topics/Alice", "topics/Alice.md"}


@pytest.mark.asyncio
async def test_parse_links_legacy_relation_text_is_ignored():
    """Old relation wrappers do not prevent their inner wikilinks from parsing."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "extends:: [[A.md]]\n\nsome [concerns:: [[B.md]]] inline\n"
        path = _write_md(tmp, "note.md", body)
        chunker = MarkdownFileChunker()
        node, _ = await chunker.chunk(path)
        assert {link.target_path for link in node.links} == {"A.md", "B.md"}


@pytest.mark.asyncio
async def test_parse_links_deduped():
    """Repeated wikilinks with the same target and anchor emit one FileLink."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "[[A.md]] again [[A.md]] and [[A.md]]"
        path = _write_md(tmp, "note.md", body)
        chunker = MarkdownFileChunker()
        node, _ = await chunker.chunk(path)
        assert len([link for link in node.links if link.target_path == "A.md"]) == 1


def test_parse_min_chunk_byte_size_clamped():
    """chunk_byte_size below 100 should be clamped to 100."""
    chunker = MarkdownFileChunker(chunk_byte_size=10)
    assert chunker.chunk_byte_size == 100


@pytest.mark.asyncio
async def test_parse_embed_toc_prefixes_chunk_text():
    """When embed_toc=True, chunks emitted inside a section are prefixed by the heading."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "# Top\n\n## Sub\n\nbody-content"
        path = _write_md(tmp, "toc.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=200, embed_toc=True)
        _, chunks = await chunker.chunk(path)
        # Single small section fits; check that the heading appears in text.
        assert any("Top" in c.text for c in chunks)


def test_parse_embed_toc_is_enabled_by_default():
    """The default adds bounded ancestor breadcrumbs without a full outline."""
    chunker = MarkdownFileChunker()
    assert chunker.embed_toc is True
    assert chunker.max_ast_sections == 100


def test_count_sections_ignores_fenced_headings_and_supports_setext():
    """The AST preflight counts real headings without parsing fenced examples."""
    content = "# Real\n\n```markdown\n# Fake\nFake too\n---\n```\n\nSetext\n===\n"
    assert MarkdownFileChunker._count_sections(content) == 2
    assert MarkdownFileChunker._count_sections(content, stop_after=1) == 2


@pytest.mark.asyncio
async def test_parse_excessive_sections_uses_plain_text_without_ast():
    """Inherited fallback is standalone, bounded, and bypasses mistletoe."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        sections = "\n\n".join(f"# Section-{i}\n\n{'x' * 480} [[target.md]]" for i in range(101))
        body = f"---\nname: fallback\n---\n{sections}"
        path = _write_md(tmp, "fallback.md", body)
        chunker = MarkdownFileChunker(
            chunk_byte_size=10000,
            embed_toc=True,
            max_ast_sections=100,
            include_frontmatter_in_metadata=True,
        )
        await chunker.start()
        try:
            with (
                patch(
                    "mistletoe.block_token.Document",
                    side_effect=AssertionError("fallback must not construct an AST"),
                ),
                patch.object(
                    chunker,
                    "chunk_content",
                    wraps=chunker.chunk_content,
                ) as chunk_content,
            ):
                node, chunks = await chunker.chunk(path)
            chunk_content.assert_called_once_with(sections, path, parse_links=True)
        finally:
            await chunker.close()

        assert isinstance(chunker, DefaultFileChunker)
        assert len(chunks) > 1
        assert max(len(chunk.text.encode("utf-8")) for chunk in chunks) <= chunker.chunk_byte_size
        assert chunks[0].start_line == 4
        assert node.front_matter.name == "fallback"
        assert node.chunk_ids == [chunk.id for chunk in chunks]
        assert {(link.target_path, link.source_path) for link in node.links} == {
            ("target.md", "fallback.md"),
        }
        assert all(chunk.metadata == {"name": "fallback"} for chunk in chunks)
        for i in (0, 100):
            assert any(f"# Section-{i}" in chunk.text for chunk in chunks)


@pytest.mark.asyncio
async def test_parse_section_limit_is_inclusive_for_ast():
    """A document at the configured section limit still takes the AST path."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        path = _write_md(tmp, "at-limit.md", "# A\n\na\n\n# B\n\nb")
        chunker = MarkdownFileChunker(max_ast_sections=2)
        with patch.object(chunker, "_build_tree", wraps=chunker._build_tree) as build_tree:
            _, chunks = await chunker.chunk(path)

        assert build_tree.call_count == 1
        assert len(chunks) == 1


@pytest.mark.asyncio
async def test_parse_small_sections_are_merged_and_headings_preserved():
    """Adjacent small sections share chunks without losing their headings."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        sections = "\n\n".join(f"## Section-{i:03d}\n\nfact-{i:03d}" for i in range(40))
        path = _write_md(tmp, "sections.md", f"# Root\n\n{sections}")
        chunker = MarkdownFileChunker(chunk_byte_size=200, embed_toc=False)
        _, chunks = await chunker.chunk(path)

        assert 1 < len(chunks) < 40
        for i in range(40):
            heading = f"## Section-{i:03d}"
            assert sum(chunk.text.count(heading) for chunk in chunks) == 1


@pytest.mark.asyncio
async def test_parse_embed_toc_uses_breadcrumbs_without_sibling_duplication():
    """TOC context repeats ancestors, not parallel section headings."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        sections = "\n\n".join(f"## Parallel-{i:03d}\n\nfact-{i:03d}" for i in range(40))
        path = _write_md(tmp, "breadcrumbs.md", f"# Root\n\n{sections}")
        chunker = MarkdownFileChunker(chunk_byte_size=200, embed_toc=True)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) > 1
        assert all("# Root" in chunk.text for chunk in chunks)
        for i in range(40):
            heading = f"## Parallel-{i:03d}"
            assert sum(chunk.text.count(heading) for chunk in chunks) == 1


@pytest.mark.asyncio
async def test_parse_heading_heavy_output_grows_linearly():
    """A thousand parallel sections remain a small, linear number of chunks."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        sections = "\n\n".join(f"## Observation-{i:04d}\n\nsynthetic fact" for i in range(1000))
        body = f"# Root\n\n{sections}"
        path = _write_md(tmp, "linear.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=10000, embed_toc=True, max_ast_sections=None)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) < 10
        assert sum(len(chunk.text) for chunk in chunks) < 2 * len(body)


@pytest.mark.asyncio
async def test_parse_nested_sections_merge_within_recursive_context():
    """Nested continuations retain their branch breadcrumb after recursive merging."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        branches = []
        for branch, fact in (("A", "a"), ("B", "b")):
            leaves = "\n\n".join(f"### {branch}{i}\n\n{fact * 45}" for i in range(4))
            branches.append(f"## {branch}\n\n{leaves}")
        branch_text = "\n\n".join(branches)
        path = _write_md(tmp, "nested.md", f"# Root\n\n{branch_text}")
        chunker = MarkdownFileChunker(chunk_byte_size=150, embed_toc=True)
        _, chunks = await chunker.chunk(path)

        assert len(chunks) == 5
        assert all(len(chunk.text.encode("utf-8")) <= chunker.chunk_byte_size for chunk in chunks)
        assert chunks[0].text == "# Root"
        assert chunks[2].text.startswith("# Root\n\n## A\n\n### A2")
        assert chunks[3].text.startswith("# Root\n\n## B\n\n### B0")
        assert "## B" not in chunks[2].text


@pytest.mark.asyncio
async def test_parse_breadcrumbs_and_part_markers_share_byte_budget():
    """Breadcrumbs and part labels cannot push multi-byte chunks over budget."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        content_lines = "\n".join(f"内容-{i}-" * 6 for i in range(4))
        body = f"# {'根' * 15}\n\n## {'枝' * 15}\n\n### {'叶' * 15}\n\n{content_lines}"
        path = _write_md(tmp, "breadcrumb-budget.md", body)
        chunker = MarkdownFileChunker(
            chunk_byte_size=100,
            embed_toc=True,
            max_ast_sections=None,
        )
        _, chunks = await chunker.chunk(path)

        assert len(chunks) > 1
        assert any(chunk.text.startswith("[Part ") for chunk in chunks)
        assert all(len(chunk.text.encode("utf-8")) <= chunker.chunk_byte_size for chunk in chunks)


@pytest.mark.asyncio
async def test_parse_frontmatter_preserves_original_line_numbers():
    """Chunk line ranges are 1-based and refer to the original file, including frontmatter."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "---\nname: t\n---\n# H\nline 1\nline 2\n"
        path = _write_md(tmp, "front-lines.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=500)
        _, chunks = await chunker.chunk(path)
        assert len(chunks) == 1
        assert chunks[0].start_line == 4
        assert chunks[0].end_line == 6


@pytest.mark.asyncio
async def test_parse_frontmatter_offsets_split_table_rows():
    """Split table row ranges include the YAML frontmatter line offset."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        rows = "".join(f"| {i} | {i} |\n" for i in range(12))
        body = "---\nname: t\n---\n| A | B |\n|---|---|\n" + rows
        path = _write_md(tmp, "front-table.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=100)
        _, chunks = await chunker.chunk(path)
        assert len(chunks) > 1
        assert chunks[0].start_line == 6
        assert chunks[0].end_line >= chunks[0].start_line


@pytest.mark.asyncio
async def test_parse_bad_frontmatter_does_not_abort_chunking():
    """Invalid YAML frontmatter is ignored while the markdown body still chunks."""

    with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
        body = "---\nname: [\n---\n# H\nbody\n"
        path = _write_md(tmp, "bad-frontmatter.md", body)
        chunker = MarkdownFileChunker(chunk_byte_size=500)
        node, chunks = await chunker.chunk(path)
        assert node.front_matter.name == ""
        assert len(chunks) == 1
        assert chunks[0].start_line == 4
        assert "body" in chunks[0].text


@pytest.mark.asyncio
async def test_chunk_does_not_split_wikilink_at_boundary():
    """A wikilink straddling the chunk_byte_size boundary should be retreated to its start."""

    # Pre-link filler is 90 bytes, link itself is 19 bytes ("[[a-very-long-target]]"=22).
    # With chunk_byte_size=100, the boundary lands inside the link.
    prefix = "x" * 90
    link = "[[a-very-long-target]]"  # 22 bytes
    suffix = "y" * 90
    content = f"# H\n{prefix}{link}{suffix}"
    with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
        f.write(content)
        temp_path = f.name

    try:
        chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=10, max_ast_sections=0)
        _, chunks = await chunker.chunk(temp_path)
        # The first chunk must NOT contain a partial link.
        first = chunks[0].text
        assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}"
        # And the link should appear intact in some chunk.
        assert any(link in c.text for c in chunks), "link was split across all chunks"
    finally:
        os.unlink(temp_path)


@pytest.mark.asyncio
async def test_chunk_does_not_split_wikilink_in_overlap():
    """A wikilink landing inside the overlap region should be advanced past."""

    # 200-byte content, chunk=100, overlap=20. First chunk ends near byte 100,
    # next start = 80. Place a link straddling byte 80 to land in the overlap.
    prefix = "a" * 75
    link = "[[overlap-target]]"  # 18 bytes; spans bytes 75..93
    suffix = "b" * 110
    content = f"# H\n{prefix}{link}{suffix}"
    with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
        f.write(content)
        temp_path = f.name

    try:
        chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=20, max_ast_sections=0)
        _, chunks = await chunker.chunk(temp_path)
        # No chunk should start mid-link.
        for c in chunks:
            t = c.text
            if "]]" in t and "[[" not in t.split("]]", 1)[0]:
                raise AssertionError(f"chunk starts mid-link: {t[:40]!r}")
        assert any(link in c.text for c in chunks)
    finally:
        os.unlink(temp_path)


@pytest.mark.asyncio
async def test_chunk_falls_back_for_oversize_link():
    """If a single link exceeds half the chunk size, the parser hard-cuts to make progress."""

    # chunk=100, link is 80 bytes, surrounded by short filler.
    # Retreating would leave a tiny chunk (< 50), so the fallback kicks in.
    prefix = "x" * 30
    link = "[[" + ("L" * 76) + "]]"  # 80 bytes total
    suffix = "y" * 200
    content = f"# H\n{prefix}{link}{suffix}"
    with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
        f.write(content)
        temp_path = f.name

    try:
        chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=10, max_ast_sections=0)
        _, chunks = await chunker.chunk(temp_path)
        # Must terminate (not hang) and cover the whole file.
        assert len(chunks) >= 2
    finally:
        os.unlink(temp_path)