diff --git a/reme/components/file_chunker/base_file_chunker.py b/reme/components/file_chunker/base_file_chunker.py index b6253e21..8ec75c8d 100644 --- a/reme/components/file_chunker/base_file_chunker.py +++ b/reme/components/file_chunker/base_file_chunker.py @@ -1,133 +1,133 @@ -"""Abstract base for file chunkers.""" - -import asyncio -import contextvars -import os -import threading -from abc import abstractmethod -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from typing import Callable - -from ..base_component import BaseComponent -from ...enumeration import ComponentEnum -from ...schema import FileChunk, FileNode -from ...utils.async_utils import wait_for_completion -from ...utils.file_utils import check_file_version - - -class BaseFileChunker(BaseComponent): - """Abstract base for file chunkers. Subclasses implement `chunk`.""" - - component_type = ComponentEnum.FILE_CHUNKER - - def __init__(self, supported_extensions: list[str] | None = None, **kwargs): - super().__init__(**kwargs) - self.supported_extensions: list[str] = supported_extensions or [] - self._worker: ThreadPoolExecutor | None = None - self._worker_lock = threading.Lock() - self._worker_state_lock = threading.Lock() - self._worker_generation = 0 - self._worker_closing = False - - async def _start(self) -> None: - await self._acquire_worker_lock() - try: - with self._worker_state_lock: - self._worker = ThreadPoolExecutor(max_workers=1, thread_name_prefix="reme-chunker") - self._worker_generation += 1 - self._worker_closing = False - finally: - self._worker_lock.release() - - async def _close(self) -> None: - with self._worker_state_lock: - self._worker_closing = True - self._worker_generation += 1 - await wait_for_completion(asyncio.create_task(self._close_worker())) - - async def _close_worker(self) -> None: - await self._acquire_worker_lock() - try: - with self._worker_state_lock: - self._worker_closing = True - self._worker_generation += 1 - worker = self._worker - self._worker = None - if worker is not None: - worker.shutdown(wait=True) - finally: - self._worker_lock.release() - - async def _acquire_worker_lock(self) -> None: - """Acquire cross-loop admission without blocking an event loop thread.""" - # Ownership spans awaited work, so a synchronous lock context would block this loop. - while not self._worker_lock.acquire(blocking=False): # pylint: disable=consider-using-with - await asyncio.sleep(0.01) - - async def _chunk_in_worker( - self, - path: str | Path, - chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], - ) -> tuple[FileNode, list[FileChunk]]: - """Admit one parse at a time without accumulating an executor work queue.""" - with self._worker_state_lock: - generation = self._worker_generation - file_path = Path(path) - rel_path = self.to_workspace_relative(path) - await self._acquire_worker_lock() - temporary_worker = False - try: - with self._worker_state_lock: - if self._worker_closing or generation != self._worker_generation: - raise RuntimeError("File chunker closed while waiting to parse") - worker = self._worker - temporary_worker = worker is None - if temporary_worker: - # Standalone chunk() remains usable without start(), with no persistent threads. - with ThreadPoolExecutor(max_workers=1, thread_name_prefix="reme-chunker") as worker: - return await self._run_chunk(worker, generation, file_path, rel_path, chunk) - assert worker is not None - return await self._run_chunk(worker, generation, file_path, rel_path, chunk) - finally: - self._worker_lock.release() - - async def _run_chunk( - self, - worker: ThreadPoolExecutor, - generation: int, - file_path: Path, - rel_path: str, - chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], - ) -> tuple[FileNode, list[FileChunk]]: - context = contextvars.copy_context() - future = asyncio.get_running_loop().run_in_executor( - worker, - context.run, - self._chunk_unchanged_file, - file_path, - rel_path, - chunk, - ) - result, version = await wait_for_completion(future) - with self._worker_state_lock: - if self._worker_closing or generation != self._worker_generation: - raise RuntimeError("File chunker closed while parsing") - check_file_version(file_path, version, file_path.stat()) - return result - - @staticmethod - def _chunk_unchanged_file( - path: Path, - rel_path: str, - chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], - ) -> tuple[tuple[FileNode, list[FileChunk]], os.stat_result]: - before = path.stat() - result = chunk(path, rel_path) - after = path.stat() - check_file_version(path, before, after) - return result, after - - @abstractmethod - async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: - """Chunk a file into (node, chunks).""" +"""Abstract base for file chunkers.""" + +import asyncio +import contextvars +import os +import threading +from abc import abstractmethod +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Callable + +from ..base_component import BaseComponent +from ...enumeration import ComponentEnum +from ...schema import FileChunk, FileNode +from ...utils.async_utils import finish_cleanup +from ...utils.file_utils import check_file_version + + +class BaseFileChunker(BaseComponent): + """Abstract base for file chunkers. Subclasses implement `chunk`.""" + + component_type = ComponentEnum.FILE_CHUNKER + + def __init__(self, supported_extensions: list[str] | None = None, **kwargs): + super().__init__(**kwargs) + self.supported_extensions: list[str] = supported_extensions or [] + self._worker: ThreadPoolExecutor | None = None + self._worker_lock = threading.Lock() + self._worker_state_lock = threading.Lock() + self._worker_generation = 0 + self._worker_closing = False + + async def _start(self) -> None: + await self._acquire_worker_lock() + try: + with self._worker_state_lock: + self._worker = ThreadPoolExecutor(max_workers=1, thread_name_prefix="reme-chunker") + self._worker_generation += 1 + self._worker_closing = False + finally: + self._worker_lock.release() + + async def _close(self) -> None: + with self._worker_state_lock: + self._worker_closing = True + self._worker_generation += 1 + await finish_cleanup(asyncio.create_task(self._close_worker())) + + async def _close_worker(self) -> None: + await self._acquire_worker_lock() + try: + with self._worker_state_lock: + self._worker_closing = True + self._worker_generation += 1 + worker = self._worker + self._worker = None + if worker is not None: + worker.shutdown(wait=True) + finally: + self._worker_lock.release() + + async def _acquire_worker_lock(self) -> None: + """Acquire cross-loop admission without blocking an event loop thread.""" + # Ownership spans awaited work, so a synchronous lock context would block this loop. + while not self._worker_lock.acquire(blocking=False): # pylint: disable=consider-using-with + await asyncio.sleep(0.01) + + async def _chunk_in_worker( + self, + path: str | Path, + chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], + ) -> tuple[FileNode, list[FileChunk]]: + """Admit one parse at a time without accumulating an executor work queue.""" + with self._worker_state_lock: + generation = self._worker_generation + file_path = Path(path) + rel_path = self.to_workspace_relative(path) + await self._acquire_worker_lock() + temporary_worker = False + try: + with self._worker_state_lock: + if self._worker_closing or generation != self._worker_generation: + raise RuntimeError("File chunker closed while waiting to parse") + worker = self._worker + temporary_worker = worker is None + if temporary_worker: + # Standalone chunk() remains usable without start(), with no persistent threads. + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="reme-chunker") as worker: + return await self._run_chunk(worker, generation, file_path, rel_path, chunk) + assert worker is not None + return await self._run_chunk(worker, generation, file_path, rel_path, chunk) + finally: + self._worker_lock.release() + + async def _run_chunk( + self, + worker: ThreadPoolExecutor, + generation: int, + file_path: Path, + rel_path: str, + chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], + ) -> tuple[FileNode, list[FileChunk]]: + context = contextvars.copy_context() + future = asyncio.get_running_loop().run_in_executor( + worker, + context.run, + self._chunk_unchanged_file, + file_path, + rel_path, + chunk, + ) + result, version = await finish_cleanup(future) + with self._worker_state_lock: + if self._worker_closing or generation != self._worker_generation: + raise RuntimeError("File chunker closed while parsing") + check_file_version(file_path, version, file_path.stat()) + return result + + @staticmethod + def _chunk_unchanged_file( + path: Path, + rel_path: str, + chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], + ) -> tuple[tuple[FileNode, list[FileChunk]], os.stat_result]: + before = path.stat() + result = chunk(path, rel_path) + after = path.stat() + check_file_version(path, before, after) + return result, after + + @abstractmethod + async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + """Chunk a file into (node, chunks).""" diff --git a/reme/components/file_chunker/markdown_file_chunker.py b/reme/components/file_chunker/markdown_file_chunker.py index 0d50d97d..9d5742c9 100644 --- a/reme/components/file_chunker/markdown_file_chunker.py +++ b/reme/components/file_chunker/markdown_file_chunker.py @@ -1,637 +1,637 @@ -"""Markdown file chunker — frontmatter + wikilink graph + AST tree chunks. - -When ``embed_toc`` is enabled, a chunk that starts inside a section carries -only that section's ancestor heading breadcrumb. Sibling headings remain in -the document stream and are stored once, avoiding the quadratic growth caused -by repeating the complete document outline in every chunk. - -Pipeline: count headings without an AST → use plain-text byte chunks when the -configured section limit is exceeded; otherwise build a mistletoe AST → -``MdNode`` tree (sections nest by heading level) → recursively chunk children -and merge adjacent small subtrees at their parent. Leaf blocks (table / code / -list / paragraph) split on internal boundaries and each piece is annotated -``[Part X/N]``. Wikilink extraction is -delegated to :class:`reme.utils.wikilink_handler.WikilinkHandler` — -the single source of truth for ``[[...]]`` syntax. -""" - -import re -import threading -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -import yaml -from pydantic import ValidationError - - -from .default_file_chunker import DefaultFileChunker, InvalidEncodingPolicy -from ..component_registry import R -from ...schema import ( - FileChunk, - FileFrontMatter, - FileNode, -) -from ...utils.wikilink_handler import WikilinkHandler - -# -- AST node + helpers --------------------------------------------------- - -_ATX_HEADING_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+|$)") -_SETEXT_HEADING_RE = re.compile(r"^ {0,3}(?:=+|-+)[ \t]*$") -_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})") - -# MarkdownRenderer changes mistletoe's process-wide token registry. -_RENDERER_LOCK = threading.Lock() - - -@dataclass -class MdNode: - """``root`` / ``section`` (heading + children until equal-or-shallower - heading) / ``body`` (one mistletoe block; ``block`` keeps the original). - - ``text`` is the rendered subtree (own heading excluded for sections). - Line ranges span the full subtree. - """ - - kind: str # "root" | "section" | "body" - heading: str | None = None - level: int = 0 - children: list["MdNode"] = field(default_factory=list) - block: Any = None - text: str = "" - start_line: int = 0 - end_line: int = 0 - - -def _heading_text(node: Any, renderer) -> str: - """Heading text without `#` markers (for outline).""" - rendered = renderer.render(node).rstrip("\n") - if rendered.startswith("#"): - return rendered.lstrip("#").strip() - return rendered.split("\n", 1)[0].strip() - - -def _finalize(n: MdNode) -> None: - """Bottom-up pass: propagate line ranges, populate ``n.text`` (rendered - subtree, own heading excluded for sections).""" - parts: list[str] = [] - for c in n.children: - _finalize(c) - if c.kind == "section": - heading = f"{'#' * c.level} {c.heading or ''}" - parts.append(f"{heading}\n\n{c.text}" if c.text else heading) - elif c.text: - parts.append(c.text) - if n.children: - first = n.children[0].start_line - n.start_line = min(n.start_line, first) if n.start_line else first - n.end_line = max(c.end_line for c in n.children) - elif n.end_line < n.start_line: - n.end_line = n.start_line - if n.kind != "body": - n.text = "\n\n".join(parts) - - -def _toc_join(*parts: str) -> str: - """Concatenate TOC fragments with ``\\n\\n``, skipping empty ones.""" - return "\n\n".join(p for p in parts if p) - - -# -- Chunker -------------------------------------------------------------- - - -@R.register("markdown") -class MarkdownFileChunker(DefaultFileChunker): - """Markdown chunker with breadcrumb context and adjacent-section packing.""" - - def __init__( - self, - encoding: str = "utf-8", - chunk_byte_size: int = 10000, - embed_toc: bool = True, - max_ast_sections: int | None = 100, - include_frontmatter_in_metadata: bool = False, - include_frontmatter_keys_in_metadata: list[str] | None = None, - *, - invalid_encoding_policy: InvalidEncodingPolicy = "replace", - **kwargs, - ): - super().__init__( - encoding=encoding, - invalid_encoding_policy=invalid_encoding_policy, - chunk_byte_size=chunk_byte_size, - **kwargs, - ) - self.embed_toc = embed_toc - self.max_ast_sections = max(0, max_ast_sections) if max_ast_sections is not None else None - self.include_frontmatter_in_metadata = include_frontmatter_in_metadata - self.include_frontmatter_keys_in_metadata = list(include_frontmatter_keys_in_metadata or []) - - def _chunk_sync(self, file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - text = self._read_text_for_indexing(file_path) - front_matter, content, line_offset = self._parse_front_matter(text) - - chunks: list[FileChunk] = [] - if content and content.strip(): - section_count = self._count_sections(content, stop_after=self.max_ast_sections) - if self.max_ast_sections is not None and section_count > self.max_ast_sections: - self.logger.info( - f"Markdown AST skipped for {rel_path}: sections>{self.max_ast_sections}; " - "using plain-text chunking", - ) - chunks = self._chunk_plain_text(content, rel_path, line_offset) - else: - from mistletoe.markdown_renderer import MarkdownRenderer - from mistletoe.block_token import Document - - with _RENDERER_LOCK, MarkdownRenderer() as renderer: - tree = self._build_tree(Document(content), renderer, line_offset=line_offset) - chunks = self._chunk_node(tree, (), rel_path, renderer) - if self.include_frontmatter_in_metadata: - chunk_metadata = self._chunk_metadata( - front_matter, - allow_keys=self.include_frontmatter_keys_in_metadata or None, - ) - for chunk in chunks: - chunk.metadata = chunk_metadata.copy() - - links = WikilinkHandler.extract_links(content, rel_path) if content else [] - - node = FileNode( - path=rel_path, - st_mtime=file_path.stat().st_mtime, - chunk_ids=[chunk.id for chunk in chunks], - links=links, - front_matter=front_matter, - ) - return node, chunks - - @staticmethod - def _count_sections(content: str, stop_after: int | None = None) -> int: - """Count block-level ATX/Setext headings without building a Markdown AST. - - Fenced code content is ignored. When ``stop_after`` is set, return as - soon as the count exceeds it because the caller only needs to choose - between AST and plain-text chunking. - """ - count = 0 - fence_char = "" - fence_length = 0 - previous_text = False - - line_start = 0 - while line_start <= len(content): - newline = content.find("\n", line_start) - if newline < 0: - line = content[line_start:] - else: - line = content[line_start:newline] - if line.endswith("\r"): - line = line[:-1] - - fence_match = _FENCE_RE.match(line) - if fence_match: - marker = fence_match.group(1) - if not fence_char: - fence_char = marker[0] - fence_length = len(marker) - elif marker[0] == fence_char and len(marker) >= fence_length and not line[fence_match.end() :].strip(): - fence_char = "" - fence_length = 0 - previous_text = False - elif not fence_char: - is_atx = _ATX_HEADING_RE.match(line) is not None - is_setext = previous_text and _SETEXT_HEADING_RE.match(line) is not None - if is_atx or is_setext: - count += 1 - if stop_after is not None and count > stop_after: - return count - previous_text = bool(line.strip()) and not is_atx and not is_setext - - if newline < 0: - break - line_start = newline + 1 - - return count - - def _chunk_plain_text(self, content: str, path: str, line_offset: int) -> list[FileChunk]: - """Use inherited byte chunking while preserving original Markdown lines.""" - chunks = self.chunk_content(content, path, parse_links=True) - if line_offset: - for chunk in chunks: - chunk.start_line += line_offset - chunk.end_line += line_offset - chunk.set_hash_id() - return chunks - - @staticmethod - def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str, int]: - """Parse YAML frontmatter, returning 1-based line offset for body AST lines. - - Invalid YAML is ignored so a single bad frontmatter block does not - prevent indexing the markdown body. - """ - lines = text.splitlines(keepends=True) - if not lines or lines[0].strip() != "---": - return FileFrontMatter(), text, 0 - - close_idx = next((i for i, line in enumerate(lines[1:], 1) if line.strip() == "---"), None) - if close_idx is None: - return FileFrontMatter(), text, 0 - - front_matter = FileFrontMatter() - try: - data = yaml.safe_load("".join(lines[1:close_idx]).strip()) or {} - if isinstance(data, dict): - front_matter = FileFrontMatter(**data) - except (yaml.YAMLError, TypeError, ValidationError): - front_matter = FileFrontMatter() - - return front_matter, "".join(lines[close_idx + 1 :]), close_idx + 1 - - @staticmethod - def _chunk_metadata( - front_matter: FileFrontMatter, - allow_keys: list[str] | None = None, - ) -> dict: - """Expose frontmatter fields on chunks so search filters can match them. - - ``allow_keys`` (when non-empty) restricts the output to that subset of - field names; absent or empty means "all fields with non-empty values". - """ - dumped = front_matter.model_dump(mode="json") - filtered = dumped if not allow_keys else {k: dumped.get(k) for k in allow_keys if k in dumped} - return {key: value for key, value in filtered.items() if value not in (None, "")} - - def _build_tree(self, doc: Any, renderer, line_offset: int = 0) -> MdNode: - """Heading-level stack folds mistletoe's flat children into nested - sections; non-headings attach as ``body`` to the current section - (or root before the first heading).""" - from mistletoe.markdown_renderer import BlankLine - from mistletoe.block_token import ( - Heading, - SetextHeading, - ) - - root = MdNode(kind="root", start_line=line_offset + 1, end_line=line_offset + 1) - stack: list[MdNode] = [root] - for child in doc.children or []: - if isinstance(child, BlankLine): - continue - raw_line = getattr(child, "line_number", None) - line = raw_line + line_offset if raw_line is not None else stack[-1].start_line - if isinstance(child, (Heading, SetextHeading)): - level = max(1, getattr(child, "level", 1)) - while len(stack) > 1 and stack[-1].level >= level: - stack.pop() - sec = MdNode( - kind="section", - heading=_heading_text(child, renderer), - level=level, - start_line=line, - ) - stack[-1].children.append(sec) - stack.append(sec) - continue - rendered = renderer.render(child).rstrip("\n") - if not rendered: - continue - stack[-1].children.append( - MdNode( - kind="body", - block=child, - text=rendered, - start_line=line, - end_line=line + rendered.count("\n"), - ), - ) - _finalize(root) - return root - - # -- Recursive subtree chunking -------------------------------------- - - def _chunk_node( - self, - node: MdNode, - ancestors: tuple[str, ...], - path: str, - renderer, - ) -> list[FileChunk]: - """Greedily assemble child subtrees, recursing only on oversized ones. - - A fitting subtree is returned immediately. For an oversized container, - fitting children are appended directly to a local ``FileChunk`` cache; - only an oversized child calls ``_chunk_node`` recursively. A full cache - is finalized before assembly continues in document order. - """ - prefix = _toc_join(*ancestors) - heading = "" - subtree_text = node.text - if node.kind == "section": - heading = f"{'#' * node.level} {node.heading or ''}" - subtree_text = _toc_join(heading, node.text) - - if subtree_text and self._byte_len(self._compose_text(prefix, subtree_text)) <= self.chunk_byte_size: - return [ - self._make_chunk( - prefix, - subtree_text, - node.start_line, - node.end_line, - path, - ), - ] - - if node.kind == "body": - return self._split_leaf(node, prefix, path, renderer) - - child_ancestors = ancestors - if node.kind == "section": - child_ancestors = (*ancestors, heading) - - chunks: list[FileChunk] = [] - cache: FileChunk | None = None - child_prefix = _toc_join(*child_ancestors) - - def flush_cache() -> None: - nonlocal cache - if cache is None: - return - chunks.append(cache.set_hash_id()) - cache = None - - def append_to_cache(text: str, start_line: int, end_line: int, new_prefix: str) -> None: - nonlocal cache - if not text: - return - if cache is not None: - candidate = _toc_join(cache.text, text) - candidate_size = self._byte_len(candidate) - if candidate_size <= self.chunk_byte_size: - cache.text = candidate - cache.end_line = end_line - if candidate_size == self.chunk_byte_size: - flush_cache() - return - flush_cache() - - cache_text = self._compose_text(new_prefix, text) - cache = FileChunk( - path=path, - start_line=start_line, - end_line=end_line, - text=cache_text, - ) - if self._byte_len(cache_text) >= self.chunk_byte_size: - flush_cache() - - if heading: - append_to_cache(heading, node.start_line, node.start_line, prefix) - - for child in node.children: - child_heading = f"{'#' * child.level} {child.heading or ''}" if child.kind == "section" else "" - child_text = _toc_join(child_heading, child.text) if child_heading else child.text - if self._byte_len(child_text) <= self.chunk_byte_size: - append_to_cache(child_text, child.start_line, child.end_line, child_prefix) - continue - - flush_cache() - chunks.extend(self._chunk_node(child, child_ancestors, path, renderer)) - - flush_cache() - return chunks - - # -- Leaf splitters: build (text, start, end) units, hand off to packer - - def _split_leaf( - self, - body: MdNode, - breadcrumb: str, - path: str, - renderer, - ) -> list[FileChunk]: - from mistletoe.block_token import ( - CodeFence, - List, - Table, - ) - - block = body.block - if isinstance(block, Table): - return self._split_table(body, breadcrumb, path) - if isinstance(block, CodeFence): - return self._split_code(body, breadcrumb, path) - if isinstance(block, List): - return self._split_list(body, breadcrumb, path, renderer) - return self._split_lines(body, breadcrumb, path) - - def _split_table( - self, - body: MdNode, - breadcrumb: str, - path: str, - ) -> list[FileChunk]: - """Repeat header + separator on every chunk.""" - from mistletoe.block_token import TableRow - - lines = body.text.split("\n") - header, data = "\n".join(lines[:2]), lines[2:] - rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)] - line_offset = body.start_line - (getattr(body.block, "line_number", None) or body.start_line) - base = body.start_line + 2 - - def line_of(i: int) -> int: - return rows[i].line_number + line_offset if i < len(rows) and rows[i].line_number else base + i - - units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)] - return self._emit_packed( - units, - breadcrumb, - path, - joiner="\n", - wrap=f"{header}\n{{inner}}", - ) - - def _split_code( - self, - body: MdNode, - breadcrumb: str, - path: str, - ) -> list[FileChunk]: - """Repeat fence opener + closer on every chunk.""" - code = body.block - indent = " " * (code.indentation or 0) - fence = f"{indent}{code.delimiter}" - opener = f"{fence}{code.info_string or ''}" - raw = (code.children[0].content if code.children else "").rstrip("\n") - if not raw: - return [] - start = body.start_line + 1 - units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))] - return self._emit_packed( - units, - breadcrumb, - path, - joiner="\n", - wrap=f"{opener}\n{{inner}}\n{fence}", - allow_empty=True, - ) - - def _split_list( - self, - body: MdNode, - breadcrumb: str, - path: str, - renderer, - ) -> list[FileChunk]: - """Pack list items; oversized items emit alone (overflow accepted).""" - from mistletoe.block_token import ListItem - - items = [c for c in (body.block.children or []) if isinstance(c, ListItem)] - if not items: - return self._split_lines(body, breadcrumb, path) - units: list[tuple[str, int, int]] = [] - line_offset = body.start_line - (getattr(body.block, "line_number", None) or body.start_line) - for it in items: - text = renderer.render(it).rstrip("\n") - if not text: - continue - line = it.line_number + line_offset if it.line_number else body.start_line - units.append((text, line, line + text.count("\n"))) - return self._emit_packed( - units, - breadcrumb, - path, - joiner="\n", - wrap="{inner}", - ) - - def _split_lines( - self, - body: MdNode, - breadcrumb: str, - path: str, - ) -> list[FileChunk]: - """Last-resort line-greedy split for paragraphs / quotes / html.""" - start = body.start_line - units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))] - return self._emit_packed( - units, - breadcrumb, - path, - joiner="\n", - wrap="{inner}", - ) - - def _emit_packed( - self, - units: list[tuple[str, int, int]], - breadcrumb: str, - path: str, - joiner: str, - wrap: str, - allow_empty: bool = False, - ) -> list[FileChunk]: - """Greedy-pack units into ``wrap`` envelopes; emit each piece. - - Envelope (table header, code fence), breadcrumb, separators and the - largest possible ``[Part X/N]`` marker all count against - ``chunk_byte_size``. Oversized atomic units overflow rather than - truncate. Multi-piece outputs get part markers; single pieces don't. - """ - envelope = self._byte_len(wrap.replace("{inner}", "")) - breadcrumb_overhead = self._breadcrumb_overhead(breadcrumb) - part_marker = len(units) > 1 - marker_overhead = self._byte_len(f"[Part {len(units)}/{len(units)}]\n\n") if part_marker else 0 - budget = self.chunk_byte_size - envelope - breadcrumb_overhead - marker_overhead - sep_len = self._byte_len(joiner) - - parts: list[tuple[str, int, int]] = [] - bucket: list[tuple[str, int, int]] = [] - bucket_bytes = 0 - - def flush() -> None: - nonlocal bucket, bucket_bytes - if not bucket: - return - inner = joiner.join(t for t, _, _ in bucket) - parts.append((inner, bucket[0][1], bucket[-1][2])) - bucket = [] - bucket_bytes = 0 - - for text, s, e in units: - if not text and not allow_empty: - continue - sep = sep_len if bucket else 0 - text_size = self._byte_len(text) - if bucket_bytes + sep + text_size > budget: - flush() - sep = 0 - bucket.append((text, s, e)) - bucket_bytes += sep + text_size - flush() - - total = len(parts) - return [ - self._make_chunk( - breadcrumb, - ( - f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}" - if total > 1 - else wrap.replace("{inner}", inner) - ), - s, - e, - path, - ) - for idx, (inner, s, e) in enumerate(parts, 1) - ] - - # -- Emit ------------------------------------------------------------- - - def _make_chunk( - self, - breadcrumb: str, - content: str, - start_line: int, - end_line: int, - path: str, - ) -> FileChunk: - """Build one chunk with an optional ancestor-heading breadcrumb.""" - text = self._compose_text(breadcrumb, content) - return FileChunk( - path=path, - start_line=start_line, - end_line=end_line, - text=text, - ).set_hash_id() - - def _breadcrumb_overhead(self, breadcrumb: str) -> int: - """Return the breadcrumb and separator cost before one content string.""" - if not self.embed_toc or not breadcrumb: - return 0 - return self._byte_len(_toc_join(breadcrumb, "x")) - 1 - - def _compose_text(self, breadcrumb: str, content: str) -> str: - """Compose a chunk and trim breadcrumbs when they consume its budget.""" - if not self.embed_toc or not breadcrumb: - return content - full = _toc_join(breadcrumb, content) - if self._byte_len(full) <= self.chunk_byte_size: - return full - - available = self.chunk_byte_size - self._byte_len(content) - 2 - if available <= 0: - return content - - breadcrumb_parts = breadcrumb.split("\n\n") - for start in range(len(breadcrumb_parts)): - retained = "\n\n".join(breadcrumb_parts[start:]) - candidate = _toc_join(retained, content) - if self._byte_len(candidate) <= self.chunk_byte_size: - return candidate - return content - - def _byte_len(self, text: str) -> int: - """Return encoded size used by the shared byte chunking contract.""" - return len(text.encode(self.encoding)) +"""Markdown file chunker — frontmatter + wikilink graph + AST tree chunks. + +When ``embed_toc`` is enabled, a chunk that starts inside a section carries +only that section's ancestor heading breadcrumb. Sibling headings remain in +the document stream and are stored once, avoiding the quadratic growth caused +by repeating the complete document outline in every chunk. + +Pipeline: count headings without an AST → use plain-text byte chunks when the +configured section limit is exceeded; otherwise build a mistletoe AST → +``MdNode`` tree (sections nest by heading level) → recursively chunk children +and merge adjacent small subtrees at their parent. Leaf blocks (table / code / +list / paragraph) split on internal boundaries and each piece is annotated +``[Part X/N]``. Wikilink extraction is +delegated to :class:`reme.utils.wikilink_handler.WikilinkHandler` — +the single source of truth for ``[[...]]`` syntax. +""" + +import re +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml +from pydantic import ValidationError + + +from .default_file_chunker import DefaultFileChunker, InvalidEncodingPolicy +from ..component_registry import R +from ...schema import ( + FileChunk, + FileFrontMatter, + FileNode, +) +from ...utils.wikilink_handler import WikilinkHandler + +# -- AST node + helpers --------------------------------------------------- + +_ATX_HEADING_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+|$)") +_SETEXT_HEADING_RE = re.compile(r"^ {0,3}(?:=+|-+)[ \t]*$") +_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})") + +# MarkdownRenderer changes mistletoe's process-wide token registry. +_RENDERER_LOCK = threading.Lock() + + +@dataclass +class MdNode: + """``root`` / ``section`` (heading + children until equal-or-shallower + heading) / ``body`` (one mistletoe block; ``block`` keeps the original). + + ``text`` is the rendered subtree (own heading excluded for sections). + Line ranges span the full subtree. + """ + + kind: str # "root" | "section" | "body" + heading: str | None = None + level: int = 0 + children: list["MdNode"] = field(default_factory=list) + block: Any = None + text: str = "" + start_line: int = 0 + end_line: int = 0 + + +def _heading_text(node: Any, renderer) -> str: + """Heading text without `#` markers (for outline).""" + rendered = renderer.render(node).rstrip("\n") + if rendered.startswith("#"): + return rendered.lstrip("#").strip() + return rendered.split("\n", 1)[0].strip() + + +def _finalize(n: MdNode) -> None: + """Bottom-up pass: propagate line ranges, populate ``n.text`` (rendered + subtree, own heading excluded for sections).""" + parts: list[str] = [] + for c in n.children: + _finalize(c) + if c.kind == "section": + heading = f"{'#' * c.level} {c.heading or ''}" + parts.append(f"{heading}\n\n{c.text}" if c.text else heading) + elif c.text: + parts.append(c.text) + if n.children: + first = n.children[0].start_line + n.start_line = min(n.start_line, first) if n.start_line else first + n.end_line = max(c.end_line for c in n.children) + elif n.end_line < n.start_line: + n.end_line = n.start_line + if n.kind != "body": + n.text = "\n\n".join(parts) + + +def _toc_join(*parts: str) -> str: + """Concatenate TOC fragments with ``\\n\\n``, skipping empty ones.""" + return "\n\n".join(p for p in parts if p) + + +# -- Chunker -------------------------------------------------------------- + + +@R.register("markdown") +class MarkdownFileChunker(DefaultFileChunker): + """Markdown chunker with breadcrumb context and adjacent-section packing.""" + + def __init__( + self, + encoding: str = "utf-8", + chunk_byte_size: int = 10000, + embed_toc: bool = True, + max_ast_sections: int | None = 100, + include_frontmatter_in_metadata: bool = False, + include_frontmatter_keys_in_metadata: list[str] | None = None, + *, + invalid_encoding_policy: InvalidEncodingPolicy = "replace", + **kwargs, + ): + super().__init__( + encoding=encoding, + invalid_encoding_policy=invalid_encoding_policy, + chunk_byte_size=chunk_byte_size, + **kwargs, + ) + self.embed_toc = embed_toc + self.max_ast_sections = max(0, max_ast_sections) if max_ast_sections is not None else None + self.include_frontmatter_in_metadata = include_frontmatter_in_metadata + self.include_frontmatter_keys_in_metadata = list(include_frontmatter_keys_in_metadata or []) + + def _chunk_sync(self, file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + text = self._read_text_for_indexing(file_path) + front_matter, content, line_offset = self._parse_front_matter(text) + + chunks: list[FileChunk] = [] + if content and content.strip(): + section_count = self._count_sections(content, stop_after=self.max_ast_sections) + if self.max_ast_sections is not None and section_count > self.max_ast_sections: + self.logger.info( + f"Markdown AST skipped for {rel_path}: sections>{self.max_ast_sections}; " + "using plain-text chunking", + ) + chunks = self._chunk_plain_text(content, rel_path, line_offset) + else: + from mistletoe.markdown_renderer import MarkdownRenderer + from mistletoe.block_token import Document + + with _RENDERER_LOCK, MarkdownRenderer() as renderer: + tree = self._build_tree(Document(content), renderer, line_offset=line_offset) + chunks = self._chunk_node(tree, (), rel_path, renderer) + if self.include_frontmatter_in_metadata: + chunk_metadata = self._chunk_metadata( + front_matter, + allow_keys=self.include_frontmatter_keys_in_metadata or None, + ) + for chunk in chunks: + chunk.metadata = chunk_metadata.copy() + + links = WikilinkHandler.extract_links(content, rel_path) if content else [] + + node = FileNode( + path=rel_path, + st_mtime=file_path.stat().st_mtime, + chunk_ids=[chunk.id for chunk in chunks], + links=links, + front_matter=front_matter, + ) + return node, chunks + + @staticmethod + def _count_sections(content: str, stop_after: int | None = None) -> int: + """Count block-level ATX/Setext headings without building a Markdown AST. + + Fenced code content is ignored. When ``stop_after`` is set, return as + soon as the count exceeds it because the caller only needs to choose + between AST and plain-text chunking. + """ + count = 0 + fence_char = "" + fence_length = 0 + previous_text = False + + line_start = 0 + while line_start <= len(content): + newline = content.find("\n", line_start) + if newline < 0: + line = content[line_start:] + else: + line = content[line_start:newline] + if line.endswith("\r"): + line = line[:-1] + + fence_match = _FENCE_RE.match(line) + if fence_match: + marker = fence_match.group(1) + if not fence_char: + fence_char = marker[0] + fence_length = len(marker) + elif marker[0] == fence_char and len(marker) >= fence_length and not line[fence_match.end() :].strip(): + fence_char = "" + fence_length = 0 + previous_text = False + elif not fence_char: + is_atx = _ATX_HEADING_RE.match(line) is not None + is_setext = previous_text and _SETEXT_HEADING_RE.match(line) is not None + if is_atx or is_setext: + count += 1 + if stop_after is not None and count > stop_after: + return count + previous_text = bool(line.strip()) and not is_atx and not is_setext + + if newline < 0: + break + line_start = newline + 1 + + return count + + def _chunk_plain_text(self, content: str, path: str, line_offset: int) -> list[FileChunk]: + """Use inherited byte chunking while preserving original Markdown lines.""" + chunks = self.chunk_content(content, path, parse_links=True) + if line_offset: + for chunk in chunks: + chunk.start_line += line_offset + chunk.end_line += line_offset + chunk.set_hash_id() + return chunks + + @staticmethod + def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str, int]: + """Parse YAML frontmatter, returning 1-based line offset for body AST lines. + + Invalid YAML is ignored so a single bad frontmatter block does not + prevent indexing the markdown body. + """ + lines = text.splitlines(keepends=True) + if not lines or lines[0].strip() != "---": + return FileFrontMatter(), text, 0 + + close_idx = next((i for i, line in enumerate(lines[1:], 1) if line.strip() == "---"), None) + if close_idx is None: + return FileFrontMatter(), text, 0 + + front_matter = FileFrontMatter() + try: + data = yaml.safe_load("".join(lines[1:close_idx]).strip()) or {} + if isinstance(data, dict): + front_matter = FileFrontMatter(**data) + except (yaml.YAMLError, TypeError, ValidationError): + front_matter = FileFrontMatter() + + return front_matter, "".join(lines[close_idx + 1 :]), close_idx + 1 + + @staticmethod + def _chunk_metadata( + front_matter: FileFrontMatter, + allow_keys: list[str] | None = None, + ) -> dict: + """Expose frontmatter fields on chunks so search filters can match them. + + ``allow_keys`` (when non-empty) restricts the output to that subset of + field names; absent or empty means "all fields with non-empty values". + """ + dumped = front_matter.model_dump(mode="json") + filtered = dumped if not allow_keys else {k: dumped.get(k) for k in allow_keys if k in dumped} + return {key: value for key, value in filtered.items() if value not in (None, "")} + + def _build_tree(self, doc: Any, renderer, line_offset: int = 0) -> MdNode: + """Heading-level stack folds mistletoe's flat children into nested + sections; non-headings attach as ``body`` to the current section + (or root before the first heading).""" + from mistletoe.markdown_renderer import BlankLine + from mistletoe.block_token import ( + Heading, + SetextHeading, + ) + + root = MdNode(kind="root", start_line=line_offset + 1, end_line=line_offset + 1) + stack: list[MdNode] = [root] + for child in doc.children or []: + if isinstance(child, BlankLine): + continue + raw_line = getattr(child, "line_number", None) + line = raw_line + line_offset if raw_line is not None else stack[-1].start_line + if isinstance(child, (Heading, SetextHeading)): + level = max(1, getattr(child, "level", 1)) + while len(stack) > 1 and stack[-1].level >= level: + stack.pop() + sec = MdNode( + kind="section", + heading=_heading_text(child, renderer), + level=level, + start_line=line, + ) + stack[-1].children.append(sec) + stack.append(sec) + continue + rendered = renderer.render(child).rstrip("\n") + if not rendered: + continue + stack[-1].children.append( + MdNode( + kind="body", + block=child, + text=rendered, + start_line=line, + end_line=line + rendered.count("\n"), + ), + ) + _finalize(root) + return root + + # -- Recursive subtree chunking -------------------------------------- + + def _chunk_node( + self, + node: MdNode, + ancestors: tuple[str, ...], + path: str, + renderer, + ) -> list[FileChunk]: + """Greedily assemble child subtrees, recursing only on oversized ones. + + A fitting subtree is returned immediately. For an oversized container, + fitting children are appended directly to a local ``FileChunk`` cache; + only an oversized child calls ``_chunk_node`` recursively. A full cache + is finalized before assembly continues in document order. + """ + prefix = _toc_join(*ancestors) + heading = "" + subtree_text = node.text + if node.kind == "section": + heading = f"{'#' * node.level} {node.heading or ''}" + subtree_text = _toc_join(heading, node.text) + + if subtree_text and self._byte_len(self._compose_text(prefix, subtree_text)) <= self.chunk_byte_size: + return [ + self._make_chunk( + prefix, + subtree_text, + node.start_line, + node.end_line, + path, + ), + ] + + if node.kind == "body": + return self._split_leaf(node, prefix, path, renderer) + + child_ancestors = ancestors + if node.kind == "section": + child_ancestors = (*ancestors, heading) + + chunks: list[FileChunk] = [] + cache: FileChunk | None = None + child_prefix = _toc_join(*child_ancestors) + + def flush_cache() -> None: + nonlocal cache + if cache is None: + return + chunks.append(cache.set_hash_id()) + cache = None + + def append_to_cache(text: str, start_line: int, end_line: int, new_prefix: str) -> None: + nonlocal cache + if not text: + return + if cache is not None: + candidate = _toc_join(cache.text, text) + candidate_size = self._byte_len(candidate) + if candidate_size <= self.chunk_byte_size: + cache.text = candidate + cache.end_line = end_line + if candidate_size == self.chunk_byte_size: + flush_cache() + return + flush_cache() + + cache_text = self._compose_text(new_prefix, text) + cache = FileChunk( + path=path, + start_line=start_line, + end_line=end_line, + text=cache_text, + ) + if self._byte_len(cache_text) >= self.chunk_byte_size: + flush_cache() + + if heading: + append_to_cache(heading, node.start_line, node.start_line, prefix) + + for child in node.children: + child_heading = f"{'#' * child.level} {child.heading or ''}" if child.kind == "section" else "" + child_text = _toc_join(child_heading, child.text) if child_heading else child.text + if self._byte_len(child_text) <= self.chunk_byte_size: + append_to_cache(child_text, child.start_line, child.end_line, child_prefix) + continue + + flush_cache() + chunks.extend(self._chunk_node(child, child_ancestors, path, renderer)) + + flush_cache() + return chunks + + # -- Leaf splitters: build (text, start, end) units, hand off to packer + + def _split_leaf( + self, + body: MdNode, + breadcrumb: str, + path: str, + renderer, + ) -> list[FileChunk]: + from mistletoe.block_token import ( + CodeFence, + List, + Table, + ) + + block = body.block + if isinstance(block, Table): + return self._split_table(body, breadcrumb, path) + if isinstance(block, CodeFence): + return self._split_code(body, breadcrumb, path) + if isinstance(block, List): + return self._split_list(body, breadcrumb, path, renderer) + return self._split_lines(body, breadcrumb, path) + + def _split_table( + self, + body: MdNode, + breadcrumb: str, + path: str, + ) -> list[FileChunk]: + """Repeat header + separator on every chunk.""" + from mistletoe.block_token import TableRow + + lines = body.text.split("\n") + header, data = "\n".join(lines[:2]), lines[2:] + rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)] + line_offset = body.start_line - (getattr(body.block, "line_number", None) or body.start_line) + base = body.start_line + 2 + + def line_of(i: int) -> int: + return rows[i].line_number + line_offset if i < len(rows) and rows[i].line_number else base + i + + units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)] + return self._emit_packed( + units, + breadcrumb, + path, + joiner="\n", + wrap=f"{header}\n{{inner}}", + ) + + def _split_code( + self, + body: MdNode, + breadcrumb: str, + path: str, + ) -> list[FileChunk]: + """Repeat fence opener + closer on every chunk.""" + code = body.block + indent = " " * (code.indentation or 0) + fence = f"{indent}{code.delimiter}" + opener = f"{fence}{code.info_string or ''}" + raw = (code.children[0].content if code.children else "").rstrip("\n") + if not raw: + return [] + start = body.start_line + 1 + units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))] + return self._emit_packed( + units, + breadcrumb, + path, + joiner="\n", + wrap=f"{opener}\n{{inner}}\n{fence}", + allow_empty=True, + ) + + def _split_list( + self, + body: MdNode, + breadcrumb: str, + path: str, + renderer, + ) -> list[FileChunk]: + """Pack list items; oversized items emit alone (overflow accepted).""" + from mistletoe.block_token import ListItem + + items = [c for c in (body.block.children or []) if isinstance(c, ListItem)] + if not items: + return self._split_lines(body, breadcrumb, path) + units: list[tuple[str, int, int]] = [] + line_offset = body.start_line - (getattr(body.block, "line_number", None) or body.start_line) + for it in items: + text = renderer.render(it).rstrip("\n") + if not text: + continue + line = it.line_number + line_offset if it.line_number else body.start_line + units.append((text, line, line + text.count("\n"))) + return self._emit_packed( + units, + breadcrumb, + path, + joiner="\n", + wrap="{inner}", + ) + + def _split_lines( + self, + body: MdNode, + breadcrumb: str, + path: str, + ) -> list[FileChunk]: + """Last-resort line-greedy split for paragraphs / quotes / html.""" + start = body.start_line + units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))] + return self._emit_packed( + units, + breadcrumb, + path, + joiner="\n", + wrap="{inner}", + ) + + def _emit_packed( + self, + units: list[tuple[str, int, int]], + breadcrumb: str, + path: str, + joiner: str, + wrap: str, + allow_empty: bool = False, + ) -> list[FileChunk]: + """Greedy-pack units into ``wrap`` envelopes; emit each piece. + + Envelope (table header, code fence), breadcrumb, separators and the + largest possible ``[Part X/N]`` marker all count against + ``chunk_byte_size``. Oversized atomic units overflow rather than + truncate. Multi-piece outputs get part markers; single pieces don't. + """ + envelope = self._byte_len(wrap.replace("{inner}", "")) + breadcrumb_overhead = self._breadcrumb_overhead(breadcrumb) + part_marker = len(units) > 1 + marker_overhead = self._byte_len(f"[Part {len(units)}/{len(units)}]\n\n") if part_marker else 0 + budget = self.chunk_byte_size - envelope - breadcrumb_overhead - marker_overhead + sep_len = self._byte_len(joiner) + + parts: list[tuple[str, int, int]] = [] + bucket: list[tuple[str, int, int]] = [] + bucket_bytes = 0 + + def flush() -> None: + nonlocal bucket, bucket_bytes + if not bucket: + return + inner = joiner.join(t for t, _, _ in bucket) + parts.append((inner, bucket[0][1], bucket[-1][2])) + bucket = [] + bucket_bytes = 0 + + for text, s, e in units: + if not text and not allow_empty: + continue + sep = sep_len if bucket else 0 + text_size = self._byte_len(text) + if bucket_bytes + sep + text_size > budget: + flush() + sep = 0 + bucket.append((text, s, e)) + bucket_bytes += sep + text_size + flush() + + total = len(parts) + return [ + self._make_chunk( + breadcrumb, + ( + f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}" + if total > 1 + else wrap.replace("{inner}", inner) + ), + s, + e, + path, + ) + for idx, (inner, s, e) in enumerate(parts, 1) + ] + + # -- Emit ------------------------------------------------------------- + + def _make_chunk( + self, + breadcrumb: str, + content: str, + start_line: int, + end_line: int, + path: str, + ) -> FileChunk: + """Build one chunk with an optional ancestor-heading breadcrumb.""" + text = self._compose_text(breadcrumb, content) + return FileChunk( + path=path, + start_line=start_line, + end_line=end_line, + text=text, + ).set_hash_id() + + def _breadcrumb_overhead(self, breadcrumb: str) -> int: + """Return the breadcrumb and separator cost before one content string.""" + if not self.embed_toc or not breadcrumb: + return 0 + return self._byte_len(_toc_join(breadcrumb, "x")) - 1 + + def _compose_text(self, breadcrumb: str, content: str) -> str: + """Compose a chunk and trim breadcrumbs when they consume its budget.""" + if not self.embed_toc or not breadcrumb: + return content + full = _toc_join(breadcrumb, content) + if self._byte_len(full) <= self.chunk_byte_size: + return full + + available = self.chunk_byte_size - self._byte_len(content) - 2 + if available <= 0: + return content + + breadcrumb_parts = breadcrumb.split("\n\n") + for start in range(len(breadcrumb_parts)): + retained = "\n\n".join(breadcrumb_parts[start:]) + candidate = _toc_join(retained, content) + if self._byte_len(candidate) <= self.chunk_byte_size: + return candidate + return content + + def _byte_len(self, text: str) -> int: + """Return encoded size used by the shared byte chunking contract.""" + return len(text.encode(self.encoding)) diff --git a/reme/steps/index/update_changes.py b/reme/steps/index/update_changes.py index 3eba6816..2ff51221 100644 --- a/reme/steps/index/update_changes.py +++ b/reme/steps/index/update_changes.py @@ -1,592 +1,558 @@ -"""Apply bounded file change batches to file_catalog or file_store.""" - -import asyncio -import math -import os -from abc import abstractmethod -from pathlib import Path -from typing import Any, Callable - -import psutil -from watchfiles import Change - -from ._change_batch import bucket_changes -from ..base_step import BaseStep -from ...components import R -from ...components.file_chunker import BaseFileChunker -from ...enumeration import ComponentEnum -from ...schema import FileChunk, FileNode -from ...utils.file_utils import FileChangedDuringChunkingError, check_file_version - -_SOURCE_CHANGE_RETRY_MARKER = "_source_changed" -_INITIAL_SCAN_RECOVERY_PASSES = 1 - - -class ChangeApplyStep(BaseStep): - """Shared added/modified/deleted handling for index update targets.""" - - target_name = "target" - reads_file_content = False - source_change_retries: int = 0 - - def __init__( - self, - persist: bool | None = None, - batch_max_files: int = 100, - batch_available_memory_ratio: float = 0.10, - batch_memory_target_bytes: int = 512 * 1024 * 1024, - batch_memory_expansion_factor: float = 8.0, - file_memory_overhead_bytes: int = 32 * 1024, - chunk_memory_overhead_bytes: int = 4 * 1024, - estimated_chunk_bytes: int = 10_000, - float16_bytes: int = 2, - **kwargs, - ): - super().__init__(**kwargs) - self.persist = persist - self.batch_max_files = int(batch_max_files) - self.batch_available_memory_ratio = float(batch_available_memory_ratio) - self.batch_memory_target_bytes = int(batch_memory_target_bytes) - self.batch_memory_expansion_factor = float(batch_memory_expansion_factor) - self.file_memory_overhead_bytes = int(file_memory_overhead_bytes) - self.chunk_memory_overhead_bytes = int(chunk_memory_overhead_bytes) - self.estimated_chunk_bytes = int(estimated_chunk_bytes) - self.float16_bytes = int(float16_bytes) - if self.batch_max_files <= 0: - raise ValueError("batch_max_files must be greater than zero") - if not math.isfinite(self.batch_available_memory_ratio) or not 0 < self.batch_available_memory_ratio <= 1: - raise ValueError("batch_available_memory_ratio must be in the interval (0, 1]") - if self.batch_memory_target_bytes <= 0: - raise ValueError("batch_memory_target_bytes must be greater than zero") - if not math.isfinite(self.batch_memory_expansion_factor) or self.batch_memory_expansion_factor <= 0: - raise ValueError("batch_memory_expansion_factor must be finite and greater than zero") - if self.file_memory_overhead_bytes < 0: - raise ValueError("file_memory_overhead_bytes must be non-negative") - if self.chunk_memory_overhead_bytes < 0: - raise ValueError("chunk_memory_overhead_bytes must be non-negative") - if self.estimated_chunk_bytes <= 0: - raise ValueError("estimated_chunk_bytes must be greater than zero") - if self.float16_bytes <= 0: - raise ValueError("float16_bytes must be greater than zero") - - @abstractmethod - async def build_item(self, path: Path) -> Any: - """Parse one existing file into the target item shape.""" - - @abstractmethod - async def upsert_items(self, items: list[Any]) -> None: - """Upsert parsed items into the target.""" - - @abstractmethod - async def delete_paths(self, paths: list[str]) -> None: - """Delete target-relative paths from the target.""" - - @abstractmethod - async def dump_target(self) -> None: - """Persist the target.""" - - async def execute(self): - assert self.context is not None - changes: list[dict] = self.context.get("changes") or [] - persist = bool(self.context.get("persist", True)) if self.persist is None else self.persist - buckets = bucket_changes( - changes, - path_exists=lambda p: self._to_abs_path(p).is_file(), - path_key=self._change_path_key, - ) - results: list[dict] = [] - pending_buckets = buckets - recover_source_changes = bool(self.context.get("retry_source_changes_before_watch")) - for recovery_pass in range(_INITIAL_SCAN_RECOVERY_PASSES + 1): - batch_results = await self._apply_existing(pending_buckets) - retryable: list[dict] = [] - for result in batch_results: - if result.get(_SOURCE_CHANGE_RETRY_MARKER, False): - retryable.append(result) - if not retryable or not recover_source_changes: - for result in batch_results: - result.pop(_SOURCE_CHANGE_RETRY_MARKER, None) - results.extend(batch_results) - break - results.extend(result for result in batch_results if not result.pop(_SOURCE_CHANGE_RETRY_MARKER, False)) - pending_buckets = bucket_changes( - [{"change": result["change"], "path": result["path"]} for result in retryable], - path_exists=lambda p: self._to_abs_path(p).is_file(), - path_key=self._change_path_key, - ) - results.extend(await self._apply_deleted(pending_buckets[Change.deleted])) - if recovery_pass == _INITIAL_SCAN_RECOVERY_PASSES: - pending_paths = pending_buckets[Change.added] + pending_buckets[Change.modified] - if pending_paths: - paths = ", ".join(pending_paths) - raise FileChangedDuringChunkingError( - f"Initial scan could not index stable source files after bounded retries: {paths}", - ) - break - results.extend(await self._apply_deleted(buckets[Change.deleted])) - if persist and results: - await self.dump_target() - self.context.response.answer = results - self.context.response.success = all(r["success"] for r in results) if results else True - return self.context.response - - async def _apply_existing(self, buckets: dict[Change, list[str]]) -> list[dict]: - results: list[dict] = [] - for change in (Change.added, Change.modified): - paths = buckets[change] - if not paths: - continue - self.logger.info(f"Detected {len(paths)} {change.name} file(s)") - items: list[Any] = [] - ok_paths: list[str] = [] - estimated_bytes = 0 - memory_budget = self._batch_memory_budget() - for path in paths: - await asyncio.sleep(0) - inspected = await self._inspect_path(change, path, results) - if inspected is None: - continue - abs_path, size_bytes = inspected - preliminary_bytes = self._try_estimate_memory( - path, - self.estimate_source_memory, - abs_path, - size_bytes, - ) - force_single_item_batch = preliminary_bytes is None - if preliminary_bytes is None: - preliminary_bytes = memory_budget + 1 - if items and self._batch_is_full(len(items), estimated_bytes, preliminary_bytes, memory_budget): - await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) - estimated_bytes = 0 - memory_budget = self._batch_memory_budget() - - built = await self._try_build_item(change, path, abs_path, size_bytes, results) - if built is None: - continue - item, size_bytes = built - del built - item_bytes = self._try_estimate_memory( - path, - self.estimate_item_memory, - item, - abs_path, - size_bytes, - ) - if item_bytes is None: - force_single_item_batch = True - item_bytes = memory_budget + 1 - elif force_single_item_batch: - item_bytes = max(item_bytes, memory_budget + 1) - if items and self._batch_is_full(len(items), estimated_bytes, item_bytes, memory_budget): - await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) - estimated_bytes = 0 - memory_budget = self._batch_memory_budget() - - items.append(item) - del item - ok_paths.append(path) - estimated_bytes += item_bytes - if self._batch_is_full(len(items), estimated_bytes, 0, memory_budget): - await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) - estimated_bytes = 0 - memory_budget = self._batch_memory_budget() - if items: - await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) - return results - - async def _inspect_path(self, change: Change, path: str, results: list[dict]) -> tuple[Path, int] | None: - abs_path = self._to_abs_path(path) - try: - if not abs_path.is_file(): - results.append( - { - "change": change.name, - "path": path, - "success": False, - "error": "not a file", - _SOURCE_CHANGE_RETRY_MARKER: True, - }, - ) - return None - size_bytes = abs_path.stat().st_size - max_file_bytes = self.max_file_bytes() - if self.reads_file_content and size_bytes > max_file_bytes: - await self.delete_paths([self.to_workspace_relative(abs_path)]) - self.logger.warning( - f"Skipping oversized file: {path} size_bytes={size_bytes} max_file_bytes={max_file_bytes}", - ) - results.append( - { - "change": change.name, - "path": path, - "success": True, - "skipped": True, - "reason": "file_too_large", - "size_bytes": size_bytes, - "max_file_bytes": max_file_bytes, - }, - ) - return None - return abs_path, size_bytes - except FileNotFoundError as e: - self.logger.debug(f"Source disappeared while processing {path}; reconciling its current state") - results.append( - { - "change": change.name, - "path": path, - "success": False, - "error": str(e), - _SOURCE_CHANGE_RETRY_MARKER: True, - }, - ) - return None - except Exception as e: - self.logger.exception(f"Failed to process {path}") - results.append({"change": change.name, "path": path, "success": False, "error": str(e)}) - return None - - async def _try_build_item( - self, - change: Change, - path: str, - abs_path: Path, - size_bytes: int, - results: list[dict], - ) -> tuple[Any, int] | None: - for attempt in range(self.source_change_retries + 1): - if attempt: - inspected = await self._inspect_path(change, path, results) - if inspected is None: - return None - abs_path, size_bytes = inspected - try: - self.logger.debug(f"Processing {change.name} file: {path}") - return await self.build_item(abs_path), size_bytes - except FileChangedDuringChunkingError as e: - if attempt < self.source_change_retries: - self.logger.debug(f"Source changed while processing {path}; retrying") - continue - self.logger.exception(f"Failed to process {path}") - results.append( - { - "change": change.name, - "path": path, - "success": False, - "error": str(e), - _SOURCE_CHANGE_RETRY_MARKER: True, - }, - ) - return None - except FileNotFoundError as e: - if attempt < self.source_change_retries: - self.logger.debug(f"Source disappeared while processing {path}; retrying") - continue - self.logger.debug(f"Source disappeared while processing {path}; reconciling its current state") - results.append( - { - "change": change.name, - "path": path, - "success": False, - "error": str(e), - _SOURCE_CHANGE_RETRY_MARKER: True, - }, - ) - return None - except Exception as e: - self.logger.exception(f"Failed to process {path}") - results.append({"change": change.name, "path": path, "success": False, "error": str(e)}) - return None - - def _try_estimate_memory( - self, - path: str, - estimate: Callable[..., int], - *args: Any, - ) -> int | None: - """Return an advisory estimate, or ``None`` so the caller can isolate the item.""" - try: - return max(0, int(estimate(*args))) - except Exception as e: - self.logger.warning(f"Failed to estimate memory for {path}; processing it alone: {e}") - return None - - def _batch_memory_budget(self) -> int: - """Return the current estimated-memory budget for a new batch.""" - try: - available = max(0, int(psutil.virtual_memory().available)) - except (AttributeError, OSError, psutil.Error) as e: - self.logger.warning(f"Failed to read available memory, using batch memory target: {e}") - return self.batch_memory_target_bytes - dynamic_budget = int(available * self.batch_available_memory_ratio) - return max(1, min(dynamic_budget, self.batch_memory_target_bytes)) - - def estimate_source_memory(self, path: Path, size_bytes: int) -> int: - """Estimate one item before reading it so the current batch can flush first.""" - del path, size_bytes - return self.file_memory_overhead_bytes - - def estimate_item_memory(self, item: Any, path: Path, size_bytes: int) -> int: - """Estimate the retained memory for one built target item.""" - del item - return self.estimate_source_memory(path, size_bytes) - - def _batch_is_full(self, item_count: int, estimated_bytes: int, next_bytes: int, memory_budget: int) -> bool: - return item_count >= self.batch_max_files or estimated_bytes + next_bytes > memory_budget - - async def _flush_upsert_batch( - self, - change: Change, - items: list[Any], - ok_paths: list[str], - estimated_bytes: int, - results: list[dict], - ) -> None: - if not items: - return - self.logger.info( - f"Applying {change.name} batch to {self.target_name}: " - f"files={len(items)} estimated_bytes={estimated_bytes}", - ) - results.extend(await self._try_upsert(change, items, ok_paths)) - items.clear() - ok_paths.clear() - - async def _try_upsert(self, change: Change, items: list[Any], ok_paths: list[str]) -> list[dict]: - try: - await self.upsert_items(items) - return [{"change": change.name, "path": p, "success": True} for p in ok_paths] - except Exception as e: - self.logger.exception(f"Failed to upsert {len(items)} {change.name} file(s) into {self.target_name}") - return [{"change": change.name, "path": p, "success": False, "error": str(e)} for p in ok_paths] - - async def _apply_deleted(self, deleted: list[str]) -> list[dict]: - if not deleted: - return [] - self.logger.info(f"Detected {len(deleted)} deleted file(s)") - results: list[dict] = [] - for start in range(0, len(deleted), self.batch_max_files): - await asyncio.sleep(0) - batch = deleted[start : start + self.batch_max_files] - try: - await self.delete_paths([self.to_workspace_relative(self._to_abs_path(p)) for p in batch]) - results.extend({"change": "deleted", "path": p, "success": True} for p in batch) - except Exception as e: - self.logger.exception(f"Failed to delete {len(batch)} file(s) from {self.target_name}") - results.extend({"change": "deleted", "path": p, "success": False, "error": str(e)} for p in batch) - return results - - def _to_abs_path(self, path: str | Path) -> Path: - p = Path(path) - return p if p.is_absolute() else self.workspace_path / p - - def _change_path_key(self, path: str) -> str: - """Return the identity used to coalesce path aliases before batching.""" - return path - - -@R.register("update_catalog_step") -class UpdateCatalogStep(ChangeApplyStep): - """Update file_catalog with a batch of file changes.""" - - target_name = "file_catalog" - - async def build_item(self, path: Path) -> FileNode: - stat = path.stat() - return FileNode(path=self.to_workspace_relative(path), st_mtime=stat.st_mtime) - - async def upsert_items(self, items: list[FileNode]) -> None: - if self.file_catalog is None: - raise RuntimeError("file_catalog is not initialized!") - await self.file_catalog.upsert(items) - - async def delete_paths(self, paths: list[str]) -> None: - if self.file_catalog is None: - raise RuntimeError("file_catalog is not initialized!") - await self.file_catalog.delete(paths) - - async def dump_target(self) -> None: - if self.file_catalog is not None: - await self.file_catalog.dump() - - -@R.register("update_index_step") -class UpdateIndexStep(ChangeApplyStep): - """Update file_store with a batch of file changes.""" - - target_name = "file_store" - reads_file_content = True - source_change_retries: int = 1 - - def __init__( - self, - persist: bool | None = None, - batch_max_files: int = 100, - batch_available_memory_ratio: float = 0.10, - batch_memory_target_bytes: int = 512 * 1024 * 1024, - batch_memory_expansion_factor: float = 8.0, - file_memory_overhead_bytes: int = 32 * 1024, - chunk_memory_overhead_bytes: int = 4 * 1024, - estimated_chunk_bytes: int = 10_000, - float16_bytes: int = 2, - **kwargs: Any, - ) -> None: - super().__init__( - persist=persist, - batch_max_files=batch_max_files, - batch_available_memory_ratio=batch_available_memory_ratio, - batch_memory_target_bytes=batch_memory_target_bytes, - batch_memory_expansion_factor=batch_memory_expansion_factor, - file_memory_overhead_bytes=file_memory_overhead_bytes, - chunk_memory_overhead_bytes=chunk_memory_overhead_bytes, - estimated_chunk_bytes=estimated_chunk_bytes, - float16_bytes=float16_bytes, - **kwargs, - ) - self._source_versions: dict[str, os.stat_result] = {} - - def _change_path_key(self, path: str) -> str: - """Coalesce absolute and workspace-relative aliases of the same file.""" - return os.path.normcase(os.fspath(self._to_abs_path(path).absolute())) - - async def _apply_existing(self, buckets: dict[Change, list[str]]) -> list[dict]: - try: - return await super()._apply_existing(buckets) - finally: - # Steps are invocation-scoped; cancelled batches must release their snapshots too. - self._source_versions.clear() - - async def build_item(self, path: Path) -> tuple[FileNode, list[FileChunk]]: - before = path.stat() - item = await self.chunk_file(path) - after = path.stat() - check_file_version(path, before, after) - self._source_versions[item[0].path] = after - return item - - async def _try_upsert( - self, - change: Change, - items: list[tuple[FileNode, list[FileChunk]]], - ok_paths: list[str], - *, - retry_source_changes: bool = True, - ) -> list[dict]: - # Recheck after waiting for the same guard used by every store mutation. - retry_paths: list[str] = [] - result_by_path: dict[str, dict] = {} - async with self.file_store._maintenance_guard(): # pylint: disable=protected-access - current_items: list[tuple[FileNode, list[FileChunk]]] = [] - current_paths: list[str] = [] - item: tuple[FileNode, list[FileChunk]] | None = None - for item, path in zip(items, ok_paths): - node = item[0] - try: - check_file_version( - self._to_abs_path(node.path), - self._source_versions[node.path], - self._to_abs_path(node.path).stat(), - ) - except (FileChangedDuringChunkingError, FileNotFoundError) as exc: - if retry_source_changes: - retry_paths.append(path) - else: - result_by_path[path] = { - "change": change.name, - "path": path, - "success": False, - "error": str(exc), - _SOURCE_CHANGE_RETRY_MARKER: True, - } - except (OSError, RuntimeError) as exc: - result_by_path[path] = { - "change": change.name, - "path": path, - "success": False, - "error": str(exc), - } - else: - current_items.append(item) - current_paths.append(path) - finally: - self._source_versions.pop(node.path) - if current_items: - result_by_path.update( - { - result["path"]: result - for result in await super()._try_upsert(change, current_items, current_paths) - }, - ) - if item is not None: - del item - items.clear() - current_items.clear() - - retry_results: list[dict] = [] - for path in retry_paths: - inspected = await self._inspect_path(change, path, retry_results) - if inspected is None: - continue - abs_path, size_bytes = inspected - built = await self._try_build_item(change, path, abs_path, size_bytes, retry_results) - if built is not None: - item, _ = built - del built - retried = await self._try_upsert( - change, - [item], - [path], - retry_source_changes=False, - ) - result_by_path.update({result["path"]: result for result in retried}) - del item - result_by_path.update({result["path"]: result for result in retry_results}) - return [result_by_path[path] for path in ok_paths] - - async def upsert_items(self, items: list[tuple[FileNode, list[FileChunk]]]) -> None: - await self.file_store.upsert(items) - - async def delete_paths(self, paths: list[str]) -> None: - await self.file_store.delete(paths) - - async def dump_target(self) -> None: - await self.file_store.dump() - - def estimate_source_memory(self, path: Path, size_bytes: int) -> int: - chunker = self._resolve_chunker(path) - chunk_bytes = max(1, int(getattr(chunker, "chunk_byte_size", self.estimated_chunk_bytes))) - estimated_chunks = max(1, (size_bytes + chunk_bytes - 1) // chunk_bytes) - return self._estimate_index_memory(size_bytes, estimated_chunks) - - def estimate_item_memory( - self, - item: tuple[FileNode, list[FileChunk]], - path: Path, - size_bytes: int, - ) -> int: - del path - return self._estimate_index_memory(size_bytes, len(item[1])) - - def _estimate_index_memory(self, size_bytes: int, chunk_count: int) -> int: - embedding_bytes = self.file_store.embedding_dimensions * self.float16_bytes - expanded_content = int(size_bytes * self.batch_memory_expansion_factor) - per_chunk = self.chunk_memory_overhead_bytes + embedding_bytes - return expanded_content + self.file_memory_overhead_bytes + max(0, chunk_count) * per_chunk - - async def chunk_file(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: - """Chunk a file into (node, chunks).""" - if self.app_context is None: - raise RuntimeError("app_context is not set when resolving file chunker") - chunker = self._resolve_chunker(Path(path)) - return await chunker.chunk(path) - - def _resolve_chunker(self, path: Path) -> BaseFileChunker: - """Resolve a file chunker for a given path.""" - chunkers: dict[str, BaseFileChunker] = self.app_context.components[ComponentEnum.FILE_CHUNKER] - suffix = path.suffix.lstrip(".").lower() - for candidate in chunkers.values(): - if suffix and suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}: - return candidate - if default := chunkers.get("default"): - return default - raise RuntimeError(f"No file chunker supports {path} (suffix={suffix!r}) and no default chunker is configured") +"""Apply bounded file change batches to file_catalog or file_store.""" + +import asyncio +import math +import os +from abc import abstractmethod +from pathlib import Path +from typing import Any, Callable + +import psutil +from watchfiles import Change + +from ._change_batch import bucket_changes +from ..base_step import BaseStep +from ...components import R +from ...components.file_chunker import BaseFileChunker +from ...enumeration import ComponentEnum +from ...schema import FileChunk, FileNode +from ...utils.file_utils import FileChangedDuringChunkingError, check_file_version + +_SOURCE_CHANGE_RETRY_MARKER = "_source_changed" +_INITIAL_SCAN_RECOVERY_PASSES = 1 + + +class ChangeApplyStep(BaseStep): + """Shared added/modified/deleted handling for index update targets.""" + + target_name = "target" + reads_file_content = False + source_change_retries: int = 0 + + def __init__( + self, + persist: bool | None = None, + batch_max_files: int = 100, + batch_available_memory_ratio: float = 0.10, + batch_memory_target_bytes: int = 512 * 1024 * 1024, + batch_memory_expansion_factor: float = 8.0, + file_memory_overhead_bytes: int = 32 * 1024, + chunk_memory_overhead_bytes: int = 4 * 1024, + estimated_chunk_bytes: int = 10_000, + float16_bytes: int = 2, + **kwargs, + ): + super().__init__(**kwargs) + self.persist = persist + self.batch_max_files = int(batch_max_files) + self.batch_available_memory_ratio = float(batch_available_memory_ratio) + self.batch_memory_target_bytes = int(batch_memory_target_bytes) + self.batch_memory_expansion_factor = float(batch_memory_expansion_factor) + self.file_memory_overhead_bytes = int(file_memory_overhead_bytes) + self.chunk_memory_overhead_bytes = int(chunk_memory_overhead_bytes) + self.estimated_chunk_bytes = int(estimated_chunk_bytes) + self.float16_bytes = int(float16_bytes) + if self.batch_max_files <= 0: + raise ValueError("batch_max_files must be greater than zero") + if not math.isfinite(self.batch_available_memory_ratio) or not 0 < self.batch_available_memory_ratio <= 1: + raise ValueError("batch_available_memory_ratio must be in the interval (0, 1]") + if self.batch_memory_target_bytes <= 0: + raise ValueError("batch_memory_target_bytes must be greater than zero") + if not math.isfinite(self.batch_memory_expansion_factor) or self.batch_memory_expansion_factor <= 0: + raise ValueError("batch_memory_expansion_factor must be finite and greater than zero") + if self.file_memory_overhead_bytes < 0: + raise ValueError("file_memory_overhead_bytes must be non-negative") + if self.chunk_memory_overhead_bytes < 0: + raise ValueError("chunk_memory_overhead_bytes must be non-negative") + if self.estimated_chunk_bytes <= 0: + raise ValueError("estimated_chunk_bytes must be greater than zero") + if self.float16_bytes <= 0: + raise ValueError("float16_bytes must be greater than zero") + + @abstractmethod + async def build_item(self, path: Path) -> Any: + """Parse one existing file into the target item shape.""" + + @abstractmethod + async def upsert_items(self, items: list[Any]) -> None: + """Upsert parsed items into the target.""" + + @abstractmethod + async def delete_paths(self, paths: list[str]) -> None: + """Delete target-relative paths from the target.""" + + @abstractmethod + async def dump_target(self) -> None: + """Persist the target.""" + + async def execute(self): + assert self.context is not None + changes: list[dict] = self.context.get("changes") or [] + persist = bool(self.context.get("persist", True)) if self.persist is None else self.persist + buckets = bucket_changes( + changes, + path_exists=lambda p: self._to_abs_path(p).is_file(), + path_key=self._change_path_key, + ) + results: list[dict] = [] + pending_buckets = buckets + recover_source_changes = bool(self.context.get("retry_source_changes_before_watch")) + for recovery_pass in range(_INITIAL_SCAN_RECOVERY_PASSES + 1): + batch_results = await self._apply_existing(pending_buckets) + retryable: list[dict] = [] + for result in batch_results: + if result.get(_SOURCE_CHANGE_RETRY_MARKER, False): + retryable.append(result) + if not retryable or not recover_source_changes: + for result in batch_results: + result.pop(_SOURCE_CHANGE_RETRY_MARKER, None) + results.extend(batch_results) + break + results.extend(result for result in batch_results if not result.pop(_SOURCE_CHANGE_RETRY_MARKER, False)) + pending_buckets = bucket_changes( + [{"change": result["change"], "path": result["path"]} for result in retryable], + path_exists=lambda p: self._to_abs_path(p).is_file(), + path_key=self._change_path_key, + ) + results.extend(await self._apply_deleted(pending_buckets[Change.deleted])) + if recovery_pass == _INITIAL_SCAN_RECOVERY_PASSES: + pending_paths = pending_buckets[Change.added] + pending_buckets[Change.modified] + if pending_paths: + paths = ", ".join(pending_paths) + raise FileChangedDuringChunkingError( + f"Initial scan could not index stable source files after bounded retries: {paths}", + ) + break + results.extend(await self._apply_deleted(buckets[Change.deleted])) + if persist and results: + await self.dump_target() + self.context.response.answer = results + self.context.response.success = all(r["success"] for r in results) if results else True + return self.context.response + + async def _apply_existing(self, buckets: dict[Change, list[str]]) -> list[dict]: + results: list[dict] = [] + for change in (Change.added, Change.modified): + paths = buckets[change] + if not paths: + continue + self.logger.info(f"Detected {len(paths)} {change.name} file(s)") + items: list[Any] = [] + ok_paths: list[str] = [] + estimated_bytes = 0 + memory_budget = self._batch_memory_budget() + for path in paths: + await asyncio.sleep(0) + inspected = await self._inspect_path(change, path, results) + if inspected is None: + continue + abs_path, size_bytes = inspected + preliminary_bytes = self._try_estimate_memory( + path, + self.estimate_source_memory, + abs_path, + size_bytes, + ) + force_single_item_batch = preliminary_bytes is None + if preliminary_bytes is None: + preliminary_bytes = memory_budget + 1 + if items and self._batch_is_full(len(items), estimated_bytes, preliminary_bytes, memory_budget): + await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) + estimated_bytes = 0 + memory_budget = self._batch_memory_budget() + + built = await self._try_build_item(change, path, abs_path, size_bytes, results) + if built is None: + continue + item, size_bytes = built + del built + item_bytes = self._try_estimate_memory( + path, + self.estimate_item_memory, + item, + abs_path, + size_bytes, + ) + if item_bytes is None: + force_single_item_batch = True + item_bytes = memory_budget + 1 + elif force_single_item_batch: + item_bytes = max(item_bytes, memory_budget + 1) + if items and self._batch_is_full(len(items), estimated_bytes, item_bytes, memory_budget): + await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) + estimated_bytes = 0 + memory_budget = self._batch_memory_budget() + + items.append(item) + del item + ok_paths.append(path) + estimated_bytes += item_bytes + if self._batch_is_full(len(items), estimated_bytes, 0, memory_budget): + await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) + estimated_bytes = 0 + memory_budget = self._batch_memory_budget() + if items: + await self._flush_upsert_batch(change, items, ok_paths, estimated_bytes, results) + return results + + async def _inspect_path(self, change: Change, path: str, results: list[dict]) -> tuple[Path, int] | None: + abs_path = self._to_abs_path(path) + try: + if not abs_path.is_file(): + results.append( + { + "change": change.name, + "path": path, + "success": False, + "error": "not a file", + _SOURCE_CHANGE_RETRY_MARKER: True, + }, + ) + return None + size_bytes = abs_path.stat().st_size + max_file_bytes = self.max_file_bytes() + if self.reads_file_content and size_bytes > max_file_bytes: + await self.delete_paths([self.to_workspace_relative(abs_path)]) + self.logger.warning( + f"Skipping oversized file: {path} size_bytes={size_bytes} max_file_bytes={max_file_bytes}", + ) + results.append( + { + "change": change.name, + "path": path, + "success": True, + "skipped": True, + "reason": "file_too_large", + "size_bytes": size_bytes, + "max_file_bytes": max_file_bytes, + }, + ) + return None + return abs_path, size_bytes + except FileNotFoundError as e: + self.logger.debug(f"Source disappeared while processing {path}; reconciling its current state") + results.append( + { + "change": change.name, + "path": path, + "success": False, + "error": str(e), + _SOURCE_CHANGE_RETRY_MARKER: True, + }, + ) + return None + except Exception as e: + self.logger.exception(f"Failed to process {path}") + results.append({"change": change.name, "path": path, "success": False, "error": str(e)}) + return None + + async def _try_build_item( + self, + change: Change, + path: str, + abs_path: Path, + size_bytes: int, + results: list[dict], + ) -> tuple[Any, int] | None: + for attempt in range(self.source_change_retries + 1): + if attempt: + inspected = await self._inspect_path(change, path, results) + if inspected is None: + return None + abs_path, size_bytes = inspected + try: + self.logger.debug(f"Processing {change.name} file: {path}") + return await self.build_item(abs_path), size_bytes + except (FileChangedDuringChunkingError, FileNotFoundError) as e: + if attempt < self.source_change_retries: + reason = "changed" if isinstance(e, FileChangedDuringChunkingError) else "disappeared" + self.logger.debug(f"Source {reason} while processing {path}; retrying") + continue + if isinstance(e, FileChangedDuringChunkingError): + self.logger.exception(f"Failed to process {path}") + else: + self.logger.debug(f"Source disappeared while processing {path}; reconciling its current state") + results.append( + { + "change": change.name, + "path": path, + "success": False, + "error": str(e), + _SOURCE_CHANGE_RETRY_MARKER: True, + }, + ) + return None + except Exception as e: + self.logger.exception(f"Failed to process {path}") + results.append({"change": change.name, "path": path, "success": False, "error": str(e)}) + return None + + def _try_estimate_memory( + self, + path: str, + estimate: Callable[..., int], + *args: Any, + ) -> int | None: + """Return an advisory estimate, or ``None`` so the caller can isolate the item.""" + try: + return max(0, int(estimate(*args))) + except Exception as e: + self.logger.warning(f"Failed to estimate memory for {path}; processing it alone: {e}") + return None + + def _batch_memory_budget(self) -> int: + """Return the current estimated-memory budget for a new batch.""" + try: + available = max(0, int(psutil.virtual_memory().available)) + except (AttributeError, OSError, psutil.Error) as e: + self.logger.warning(f"Failed to read available memory, using batch memory target: {e}") + return self.batch_memory_target_bytes + dynamic_budget = int(available * self.batch_available_memory_ratio) + return max(1, min(dynamic_budget, self.batch_memory_target_bytes)) + + def estimate_source_memory(self, path: Path, size_bytes: int) -> int: + """Estimate one item before reading it so the current batch can flush first.""" + del path, size_bytes + return self.file_memory_overhead_bytes + + def estimate_item_memory(self, item: Any, path: Path, size_bytes: int) -> int: + """Estimate the retained memory for one built target item.""" + del item + return self.estimate_source_memory(path, size_bytes) + + def _batch_is_full(self, item_count: int, estimated_bytes: int, next_bytes: int, memory_budget: int) -> bool: + return item_count >= self.batch_max_files or estimated_bytes + next_bytes > memory_budget + + async def _flush_upsert_batch( + self, + change: Change, + items: list[Any], + ok_paths: list[str], + estimated_bytes: int, + results: list[dict], + ) -> None: + if not items: + return + self.logger.info( + f"Applying {change.name} batch to {self.target_name}: " + f"files={len(items)} estimated_bytes={estimated_bytes}", + ) + results.extend(await self._try_upsert(change, items, ok_paths)) + items.clear() + ok_paths.clear() + + async def _try_upsert(self, change: Change, items: list[Any], ok_paths: list[str]) -> list[dict]: + try: + await self.upsert_items(items) + return [{"change": change.name, "path": p, "success": True} for p in ok_paths] + except Exception as e: + self.logger.exception(f"Failed to upsert {len(items)} {change.name} file(s) into {self.target_name}") + return [{"change": change.name, "path": p, "success": False, "error": str(e)} for p in ok_paths] + + async def _apply_deleted(self, deleted: list[str]) -> list[dict]: + if not deleted: + return [] + self.logger.info(f"Detected {len(deleted)} deleted file(s)") + results: list[dict] = [] + for start in range(0, len(deleted), self.batch_max_files): + await asyncio.sleep(0) + batch = deleted[start : start + self.batch_max_files] + try: + await self.delete_paths([self.to_workspace_relative(self._to_abs_path(p)) for p in batch]) + results.extend({"change": "deleted", "path": p, "success": True} for p in batch) + except Exception as e: + self.logger.exception(f"Failed to delete {len(batch)} file(s) from {self.target_name}") + results.extend({"change": "deleted", "path": p, "success": False, "error": str(e)} for p in batch) + return results + + def _to_abs_path(self, path: str | Path) -> Path: + p = Path(path) + return p if p.is_absolute() else self.workspace_path / p + + def _change_path_key(self, path: str) -> str: + """Return the identity used to coalesce path aliases before batching.""" + return path + + +@R.register("update_catalog_step") +class UpdateCatalogStep(ChangeApplyStep): + """Update file_catalog with a batch of file changes.""" + + target_name = "file_catalog" + + async def build_item(self, path: Path) -> FileNode: + stat = path.stat() + return FileNode(path=self.to_workspace_relative(path), st_mtime=stat.st_mtime) + + async def upsert_items(self, items: list[FileNode]) -> None: + if self.file_catalog is None: + raise RuntimeError("file_catalog is not initialized!") + await self.file_catalog.upsert(items) + + async def delete_paths(self, paths: list[str]) -> None: + if self.file_catalog is None: + raise RuntimeError("file_catalog is not initialized!") + await self.file_catalog.delete(paths) + + async def dump_target(self) -> None: + if self.file_catalog is not None: + await self.file_catalog.dump() + + +@R.register("update_index_step") +class UpdateIndexStep(ChangeApplyStep): + """Update file_store with a batch of file changes.""" + + target_name = "file_store" + reads_file_content = True + source_change_retries: int = 1 + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._source_versions: dict[str, os.stat_result] = {} + + def _change_path_key(self, path: str) -> str: + """Coalesce absolute and workspace-relative aliases of the same file.""" + return os.path.normcase(os.fspath(self._to_abs_path(path).absolute())) + + async def _apply_existing(self, buckets: dict[Change, list[str]]) -> list[dict]: + try: + return await super()._apply_existing(buckets) + finally: + # Steps are invocation-scoped; cancelled batches must release their snapshots too. + self._source_versions.clear() + + async def build_item(self, path: Path) -> tuple[FileNode, list[FileChunk]]: + before = path.stat() + item = await self.chunk_file(path) + after = path.stat() + check_file_version(path, before, after) + self._source_versions[item[0].path] = after + return item + + async def _try_upsert( + self, + change: Change, + items: list[tuple[FileNode, list[FileChunk]]], + ok_paths: list[str], + *, + retry_source_changes: bool = True, + ) -> list[dict]: + # Recheck after waiting for the same guard used by every store mutation. + retry_paths: list[str] = [] + result_by_path: dict[str, dict] = {} + async with self.file_store._maintenance_guard(): # pylint: disable=protected-access + current_items: list[tuple[FileNode, list[FileChunk]]] = [] + current_paths: list[str] = [] + item: tuple[FileNode, list[FileChunk]] | None = None + for item, path in zip(items, ok_paths): + node = item[0] + try: + check_file_version( + self._to_abs_path(node.path), + self._source_versions[node.path], + self._to_abs_path(node.path).stat(), + ) + except (FileChangedDuringChunkingError, FileNotFoundError) as exc: + if retry_source_changes: + retry_paths.append(path) + else: + result_by_path[path] = { + "change": change.name, + "path": path, + "success": False, + "error": str(exc), + _SOURCE_CHANGE_RETRY_MARKER: True, + } + except (OSError, RuntimeError) as exc: + result_by_path[path] = { + "change": change.name, + "path": path, + "success": False, + "error": str(exc), + } + else: + current_items.append(item) + current_paths.append(path) + finally: + self._source_versions.pop(node.path) + if current_items: + result_by_path.update( + { + result["path"]: result + for result in await super()._try_upsert(change, current_items, current_paths) + }, + ) + if item is not None: + del item + items.clear() + current_items.clear() + + retry_results: list[dict] = [] + for path in retry_paths: + inspected = await self._inspect_path(change, path, retry_results) + if inspected is None: + continue + abs_path, size_bytes = inspected + built = await self._try_build_item(change, path, abs_path, size_bytes, retry_results) + if built is not None: + item, _ = built + del built + retried = await self._try_upsert( + change, + [item], + [path], + retry_source_changes=False, + ) + result_by_path.update({result["path"]: result for result in retried}) + del item + result_by_path.update({result["path"]: result for result in retry_results}) + return [result_by_path[path] for path in ok_paths] + + async def upsert_items(self, items: list[tuple[FileNode, list[FileChunk]]]) -> None: + await self.file_store.upsert(items) + + async def delete_paths(self, paths: list[str]) -> None: + await self.file_store.delete(paths) + + async def dump_target(self) -> None: + await self.file_store.dump() + + def estimate_source_memory(self, path: Path, size_bytes: int) -> int: + chunker = self._resolve_chunker(path) + chunk_bytes = max(1, int(getattr(chunker, "chunk_byte_size", self.estimated_chunk_bytes))) + estimated_chunks = max(1, (size_bytes + chunk_bytes - 1) // chunk_bytes) + return self._estimate_index_memory(size_bytes, estimated_chunks) + + def estimate_item_memory( + self, + item: tuple[FileNode, list[FileChunk]], + path: Path, + size_bytes: int, + ) -> int: + del path + return self._estimate_index_memory(size_bytes, len(item[1])) + + def _estimate_index_memory(self, size_bytes: int, chunk_count: int) -> int: + embedding_bytes = self.file_store.embedding_dimensions * self.float16_bytes + expanded_content = int(size_bytes * self.batch_memory_expansion_factor) + per_chunk = self.chunk_memory_overhead_bytes + embedding_bytes + return expanded_content + self.file_memory_overhead_bytes + max(0, chunk_count) * per_chunk + + async def chunk_file(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + """Chunk a file into (node, chunks).""" + if self.app_context is None: + raise RuntimeError("app_context is not set when resolving file chunker") + chunker = self._resolve_chunker(Path(path)) + return await chunker.chunk(path) + + def _resolve_chunker(self, path: Path) -> BaseFileChunker: + """Resolve a file chunker for a given path.""" + chunkers: dict[str, BaseFileChunker] = self.app_context.components[ComponentEnum.FILE_CHUNKER] + suffix = path.suffix.lstrip(".").lower() + for candidate in chunkers.values(): + if suffix and suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}: + return candidate + if default := chunkers.get("default"): + return default + raise RuntimeError(f"No file chunker supports {path} (suffix={suffix!r}) and no default chunker is configured") diff --git a/reme/utils/async_utils.py b/reme/utils/async_utils.py index 3bb74a5c..d2dd9a85 100644 --- a/reme/utils/async_utils.py +++ b/reme/utils/async_utils.py @@ -10,16 +10,7 @@ T = TypeVar("T") async def finish_cleanup(operation: Awaitable[T]) -> T: """Finish cleanup before propagating any cancellation received while waiting.""" - return await wait_for_completion(asyncio.ensure_future(operation)) - - -async def complete_in_thread(func: Callable[..., Any], /, *args) -> Any: - """Finish a side-effecting thread call, including through repeated cancellation.""" - return await finish_cleanup(asyncio.to_thread(func, *args)) - - -async def wait_for_completion(task: asyncio.Future[T]) -> T: - """Wait for work to finish before propagating the first cancellation.""" + task = asyncio.ensure_future(operation) cancellation: asyncio.CancelledError | None = None while not task.done(): try: @@ -36,3 +27,8 @@ async def wait_for_completion(task: asyncio.Future[T]) -> T: task.result() raise cancellation return task.result() + + +async def complete_in_thread(func: Callable[..., Any], /, *args) -> Any: + """Finish a side-effecting thread call, including through repeated cancellation.""" + return await finish_cleanup(asyncio.to_thread(func, *args)) diff --git a/tests/unit/test_file_chunker_workers.py b/tests/unit/test_file_chunker_workers.py index d916515c..fb9e2697 100644 --- a/tests/unit/test_file_chunker_workers.py +++ b/tests/unit/test_file_chunker_workers.py @@ -1,822 +1,822 @@ -"""Worker admission, output compatibility, and real indexing/search regressions.""" - -import asyncio -import contextvars -import gc -import json -import os -import threading -import weakref -from contextlib import asynccontextmanager -from pathlib import Path -from typing import Callable -from unittest.mock import patch - -import pytest -from httpx import ASGITransport, AsyncClient -from watchfiles import Change - -from reme.application import Application -from reme.components.application_context import ApplicationContext -from reme.components.file_chunker import DefaultFileChunker, JsonFileChunker, JsonlFileChunker, MarkdownFileChunker -from reme.components.file_store import LocalFileStore -from reme.components.job import BaseJob, StreamJob -from reme.components.runtime_context import RuntimeContext -from reme.components.service import HttpService -from reme.enumeration import ChunkEnum, ComponentEnum -from reme.schema import FileChunk, FileNode, Response, StreamChunk -from reme.steps.index import UpdateIndexStep - -# Worker tests intentionally inspect admission and teardown at the internal boundary. -# pylint: disable=protected-access - - -class _StoreSearchJob(BaseJob): - """Use the actual keyword store behind the HTTP route and MCP tool.""" - - def __init__(self, store: LocalFileStore) -> None: - super().__init__(name="search", parameters={"type": "object", "properties": {"query": {"type": "string"}}}) - self.store = store - - async def __call__(self, query: str, **kwargs) -> Response: - del kwargs - chunks = await self.store.keyword_search(query, limit=10, search_filter={}) - return Response(answer=[chunk.text for chunk in chunks]) - - -class _StoreSearchStreamJob(StreamJob): - """Stream real keyword matches through the existing SSE endpoint.""" - - def __init__(self, store: LocalFileStore) -> None: - super().__init__(name="search_stream") - self.store = store - - async def __call__(self, stream_queue: asyncio.Queue[StreamChunk], query: str, **kwargs) -> None: - del kwargs - chunks = await self.store.keyword_search(query, limit=10, search_filter={}) - for chunk in chunks: - await stream_queue.put(StreamChunk(chunk=chunk.text)) - await asyncio.sleep(0) - await stream_queue.put(StreamChunk(chunk_type=ChunkEnum.DONE, done=True)) - - -@pytest.mark.asyncio -async def test_outputs_equal_pre_worker_fixtures(tmp_path: Path) -> None: - """Compare every serialized field, including hashes, links, metadata and ranges.""" - context = ApplicationContext(workspace_dir=str(tmp_path)) - cases = [ - (DefaultFileChunker(chunk_byte_size=100), "plain.txt", "first\r\n" + "abc " * 100), - ( - MarkdownFileChunker(chunk_byte_size=100, include_frontmatter_in_metadata=True), - "note.md", - "---\ntitle: Example\ntags: [one, two]\n---\n# First\n\nbody [[other.md]]\n\n## Second\n\n" + "text " * 70, - ), - (MarkdownFileChunker(chunk_byte_size=100, max_ast_sections=0), "fallback.md", "# First\n" + "text " * 100), - ( - JsonFileChunker(chunk_chars=256), - "data.json", - json.dumps({"items": [{"name": "abc" * 35, "n": i} for i in range(6)]}, indent=2), - ), - (JsonFileChunker(chunk_chars=256), "bad.json", "{invalid\ntext\n"), - (JsonlFileChunker(max_chars=64, max_overlap_chars=30), "events.jsonl", '{"value": "alpha"}\n' * 12), - ] - expected = json.loads((Path(__file__).parent / "fixtures" / "file_chunker_outputs.json").read_text()) - for chunker, name, text in cases: - chunker.app_context = context - path = tmp_path / name - path.write_bytes(text.encode()) - node, chunks = await chunker.chunk(path) - output = {"node": node.model_dump(mode="json"), "chunks": [c.model_dump(mode="json") for c in chunks]} - assert output["node"].pop("st_mtime") == path.stat().st_mtime - assert output == expected[name] - - -async def _wait_started(event: threading.Event) -> None: - for _ in range(500): - if event.is_set(): - return - await asyncio.sleep(0.01) - raise AssertionError("worker did not start") - - -@pytest.mark.asyncio -@pytest.mark.parametrize("chunker_class", [DefaultFileChunker, MarkdownFileChunker, JsonFileChunker, JsonlFileChunker]) -async def test_parsing_leaves_event_loop_responsive( - tmp_path: Path, - chunker_class: type[DefaultFileChunker | MarkdownFileChunker | JsonFileChunker | JsonlFileChunker], -) -> None: - """A parser can block while unrelated async requests keep progressing.""" - chunker = chunker_class() - path = tmp_path / "source" - path.write_text("{}") - entered, release = threading.Event(), threading.Event() - main_thread = threading.get_ident() - original = chunker._chunk_sync - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - assert threading.get_ident() != main_thread - entered.set() - assert release.wait(5) - return original(file_path, rel_path) - - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - task = asyncio.create_task(chunker.chunk(path)) - try: - await _wait_started(entered) - for _ in range(10): - await asyncio.sleep(0) - assert not task.done() - finally: - release.set() - await task - assert chunker._worker is None - - -@pytest.mark.asyncio -async def test_cancellation_drains_worker_and_preserves_admission(tmp_path: Path) -> None: - """Repeated cancellation cannot free admission while parsing is still running.""" - path = tmp_path / "source.txt" - path.write_text("source") - chunker = DefaultFileChunker() - entered, release = threading.Event(), threading.Event() - calls = [] - original = chunker._chunk_sync - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - calls.append(rel_path) - entered.set() - assert release.wait(5) - return original(file_path, rel_path) - - async with chunker: - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - first = asyncio.create_task(chunker.chunk(path)) - await _wait_started(entered) - first.cancel("original cancellation") - second = asyncio.create_task(chunker.chunk(path)) - await asyncio.sleep(0.01) - first.cancel("second cancellation") - await asyncio.sleep(0.01) - try: - assert not first.done() - assert len(calls) == 1 - finally: - release.set() - with pytest.raises(asyncio.CancelledError, match="original cancellation"): - await first - await second - assert chunker._worker is None - assert len(calls) == 2 - - -@pytest.mark.asyncio -async def test_relative_delete_alias_removes_indexed_content(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A relative-first alias batch resolves against the workspace, not the working directory.""" - workspace = tmp_path / "workspace" - workspace.mkdir() - monkeypatch.chdir(tmp_path) - source = workspace / "source.txt" - source.write_text("unique searchable content") - context = ApplicationContext(workspace_dir=str(workspace)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - async with LocalFileStore(embedding_store="") as store, chunker: - await store.upsert([await chunker.chunk(source)]) - assert await store.keyword_search("unique", limit=10, search_filter={}) - source.unlink() - step = UpdateIndexStep(file_store=store, persist=False, app_context=context) +"""Worker admission, output compatibility, and real indexing/search regressions.""" + +import asyncio +import contextvars +import gc +import json +import os +import threading +import weakref +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Callable +from unittest.mock import patch + +import pytest +from httpx import ASGITransport, AsyncClient +from watchfiles import Change + +from reme.application import Application +from reme.components.application_context import ApplicationContext +from reme.components.file_chunker import DefaultFileChunker, JsonFileChunker, JsonlFileChunker, MarkdownFileChunker +from reme.components.file_store import LocalFileStore +from reme.components.job import BaseJob, StreamJob +from reme.components.runtime_context import RuntimeContext +from reme.components.service import HttpService +from reme.enumeration import ChunkEnum, ComponentEnum +from reme.schema import FileChunk, FileNode, Response, StreamChunk +from reme.steps.index import UpdateIndexStep + +# Worker tests intentionally inspect admission and teardown at the internal boundary. +# pylint: disable=protected-access + + +class _StoreSearchJob(BaseJob): + """Use the actual keyword store behind the HTTP route and MCP tool.""" + + def __init__(self, store: LocalFileStore) -> None: + super().__init__(name="search", parameters={"type": "object", "properties": {"query": {"type": "string"}}}) + self.store = store + + async def __call__(self, query: str, **kwargs) -> Response: + del kwargs + chunks = await self.store.keyword_search(query, limit=10, search_filter={}) + return Response(answer=[chunk.text for chunk in chunks]) + + +class _StoreSearchStreamJob(StreamJob): + """Stream real keyword matches through the existing SSE endpoint.""" + + def __init__(self, store: LocalFileStore) -> None: + super().__init__(name="search_stream") + self.store = store + + async def __call__(self, stream_queue: asyncio.Queue[StreamChunk], query: str, **kwargs) -> None: + del kwargs + chunks = await self.store.keyword_search(query, limit=10, search_filter={}) + for chunk in chunks: + await stream_queue.put(StreamChunk(chunk=chunk.text)) + await asyncio.sleep(0) + await stream_queue.put(StreamChunk(chunk_type=ChunkEnum.DONE, done=True)) + + +@pytest.mark.asyncio +async def test_outputs_equal_pre_worker_fixtures(tmp_path: Path) -> None: + """Compare every serialized field, including hashes, links, metadata and ranges.""" + context = ApplicationContext(workspace_dir=str(tmp_path)) + cases = [ + (DefaultFileChunker(chunk_byte_size=100), "plain.txt", "first\r\n" + "abc " * 100), + ( + MarkdownFileChunker(chunk_byte_size=100, include_frontmatter_in_metadata=True), + "note.md", + "---\ntitle: Example\ntags: [one, two]\n---\n# First\n\nbody [[other.md]]\n\n## Second\n\n" + "text " * 70, + ), + (MarkdownFileChunker(chunk_byte_size=100, max_ast_sections=0), "fallback.md", "# First\n" + "text " * 100), + ( + JsonFileChunker(chunk_chars=256), + "data.json", + json.dumps({"items": [{"name": "abc" * 35, "n": i} for i in range(6)]}, indent=2), + ), + (JsonFileChunker(chunk_chars=256), "bad.json", "{invalid\ntext\n"), + (JsonlFileChunker(max_chars=64, max_overlap_chars=30), "events.jsonl", '{"value": "alpha"}\n' * 12), + ] + expected = json.loads((Path(__file__).parent / "fixtures" / "file_chunker_outputs.json").read_text()) + for chunker, name, text in cases: + chunker.app_context = context + path = tmp_path / name + path.write_bytes(text.encode()) + node, chunks = await chunker.chunk(path) + output = {"node": node.model_dump(mode="json"), "chunks": [c.model_dump(mode="json") for c in chunks]} + assert output["node"].pop("st_mtime") == path.stat().st_mtime + assert output == expected[name] + + +async def _wait_started(event: threading.Event) -> None: + for _ in range(500): + if event.is_set(): + return + await asyncio.sleep(0.01) + raise AssertionError("worker did not start") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunker_class", [DefaultFileChunker, MarkdownFileChunker, JsonFileChunker, JsonlFileChunker]) +async def test_parsing_leaves_event_loop_responsive( + tmp_path: Path, + chunker_class: type[DefaultFileChunker | MarkdownFileChunker | JsonFileChunker | JsonlFileChunker], +) -> None: + """A parser can block while unrelated async requests keep progressing.""" + chunker = chunker_class() + path = tmp_path / "source" + path.write_text("{}") + entered, release = threading.Event(), threading.Event() + main_thread = threading.get_ident() + original = chunker._chunk_sync + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + assert threading.get_ident() != main_thread + entered.set() + assert release.wait(5) + return original(file_path, rel_path) + + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + task = asyncio.create_task(chunker.chunk(path)) + try: + await _wait_started(entered) + for _ in range(10): + await asyncio.sleep(0) + assert not task.done() + finally: + release.set() + await task + assert chunker._worker is None + + +@pytest.mark.asyncio +async def test_cancellation_drains_worker_and_preserves_admission(tmp_path: Path) -> None: + """Repeated cancellation cannot free admission while parsing is still running.""" + path = tmp_path / "source.txt" + path.write_text("source") + chunker = DefaultFileChunker() + entered, release = threading.Event(), threading.Event() + calls = [] + original = chunker._chunk_sync + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + calls.append(rel_path) + entered.set() + assert release.wait(5) + return original(file_path, rel_path) + + async with chunker: + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + first = asyncio.create_task(chunker.chunk(path)) + await _wait_started(entered) + first.cancel("original cancellation") + second = asyncio.create_task(chunker.chunk(path)) + await asyncio.sleep(0.01) + first.cancel("second cancellation") + await asyncio.sleep(0.01) + try: + assert not first.done() + assert len(calls) == 1 + finally: + release.set() + with pytest.raises(asyncio.CancelledError, match="original cancellation"): + await first + await second + assert chunker._worker is None + assert len(calls) == 2 + + +@pytest.mark.asyncio +async def test_relative_delete_alias_removes_indexed_content(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A relative-first alias batch resolves against the workspace, not the working directory.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + monkeypatch.chdir(tmp_path) + source = workspace / "source.txt" + source.write_text("unique searchable content") + context = ApplicationContext(workspace_dir=str(workspace)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + async with LocalFileStore(embedding_store="") as store, chunker: + await store.upsert([await chunker.chunk(source)]) + assert await store.keyword_search("unique", limit=10, search_filter={}) + source.unlink() + step = UpdateIndexStep(file_store=store, persist=False, app_context=context) response = await step( RuntimeContext( - changes=[ - {"change": "deleted", "path": "source.txt"}, - {"change": "deleted", "path": str(source)}, + changes=[ + {"change": "deleted", "path": "source.txt"}, + {"change": "deleted", "path": str(source)}, ], ), - ) - assert response.success is True - assert await store.get_nodes() == [] - assert not await store.keyword_search("unique", limit=10, search_filter={}) - - -@pytest.mark.asyncio -async def test_shutdown_rejects_queued_work_and_drains_despite_cancellation(tmp_path: Path) -> None: - """Closing waits for running work, rejects queued parsing, and permits restart.""" - path = tmp_path / "source.txt" - path.write_text("source") - chunker = DefaultFileChunker() - entered, release = threading.Event(), threading.Event() - original = chunker._chunk_sync - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - entered.set() - assert release.wait(5) - return original(file_path, rel_path) - - await chunker.start() - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - running = asyncio.create_task(chunker.chunk(path)) - await _wait_started(entered) - queued = asyncio.create_task(chunker.chunk(path)) - await asyncio.sleep(0) - closing = asyncio.create_task(chunker.close()) - await asyncio.sleep(0.01) - closing.cancel() - try: - assert not closing.done() - finally: - release.set() - with pytest.raises(RuntimeError, match="closed"): - await running - with pytest.raises(RuntimeError, match="closed"): - await queued - with pytest.raises(asyncio.CancelledError): - await closing - assert chunker._worker is None - assert not chunker.is_started - async with chunker: - assert (await chunker.chunk(path))[1][0].text == "source" - - -@pytest.mark.asyncio -async def test_shutdown_from_another_event_loop_waits_for_worker_and_closes_executor(tmp_path: Path) -> None: - """Worker admission and shutdown can cross event loops without stranding the executor.""" - path = tmp_path / "source.txt" - path.write_text("source") - chunker = DefaultFileChunker() - entered, release = threading.Event(), threading.Event() - worker_errors: list[BaseException] = [] - original = chunker._chunk_sync - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - entered.set() - assert release.wait(5) - return original(file_path, rel_path) - - def run_on_worker_loop() -> None: - try: - asyncio.run(chunker.chunk(path)) - except BaseException as exc: - worker_errors.append(exc) - - await chunker.start() - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - worker_thread = threading.Thread(target=run_on_worker_loop) - worker_thread.start() - await _wait_started(entered) - closing = asyncio.create_task(chunker.close()) - try: - for _ in range(100): - if chunker._worker_closing: - break - await asyncio.sleep(0.01) - assert chunker._worker_closing - assert not closing.done() - finally: - release.set() - await closing - await asyncio.to_thread(worker_thread.join, 5) - - assert not worker_thread.is_alive() - assert len(worker_errors) == 1 - assert isinstance(worker_errors[0], RuntimeError) - assert "different event loop" not in str(worker_errors[0]) - assert chunker._worker is None - assert not chunker.is_started - - -@pytest.mark.asyncio -@pytest.mark.parametrize("change", ["modify", "replace", "delete"]) -async def test_source_changes_do_not_return_stale_chunks(tmp_path: Path, change: str) -> None: - """Changes after reading cannot be mistaken for successfully parsed current content.""" - path = tmp_path / "source.txt" - path.write_text("old source") - chunker = DefaultFileChunker() - original = chunker._chunk_sync - entered, release = threading.Event(), threading.Event() - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - result = original(file_path, rel_path) - entered.set() - assert release.wait(5) - return result - - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - task = asyncio.create_task(chunker.chunk(path)) - await _wait_started(entered) - try: - if change == "modify": - path.write_text("new source with different size") - elif change == "replace": - replacement = tmp_path / "replacement" - replacement.write_text("replacement source") - replacement.replace(path) - else: - path.unlink() - finally: - release.set() - with pytest.raises((RuntimeError, FileNotFoundError)): - await task - - -@pytest.mark.asyncio -async def test_change_before_event_loop_receives_result_is_rejected(tmp_path: Path) -> None: - """Validate the worker snapshot again when its result reaches the calling loop.""" - path = tmp_path / "source.txt" - path.write_text("old source") - chunker = DefaultFileChunker() - original = chunker._chunk_unchanged_file - loop = asyncio.get_running_loop() - changed = threading.Event() - - def change_on_loop() -> None: - path.write_text("changed after the worker snapshot") - changed.set() - - def change_after_parse( - file_path: Path, - rel_path: str, - chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], - ) -> tuple[tuple[FileNode, list[FileChunk]], os.stat_result]: - result = original(file_path, rel_path, chunk) - loop.call_soon_threadsafe(change_on_loop) - assert changed.wait(5) - return result - - with patch.object(chunker, "_chunk_unchanged_file", side_effect=change_after_parse): - with pytest.raises(RuntimeError, match="File changed"): - await chunker.chunk(path) - - -@pytest.mark.asyncio -async def test_queued_cancellation_does_not_submit_and_context_is_preserved(tmp_path: Path) -> None: - """A cancelled waiter uses no worker slot; caller context is copied into the worker.""" - path = tmp_path / "source.txt" - path.write_text("source") - chunker = DefaultFileChunker() - entered, release = threading.Event(), threading.Event() - marker = contextvars.ContextVar("chunk_test_marker", default="unset") - original = chunker._chunk_sync - calls = [] - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - calls.append(marker.get()) - entered.set() - assert release.wait(5) - return original(file_path, rel_path) - - token = marker.set("caller") - try: - async with chunker: - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - first = asyncio.create_task(chunker.chunk(path)) - await _wait_started(entered) - queued = asyncio.create_task(chunker.chunk(path)) - await asyncio.sleep(0) - queued.cancel() - try: - with pytest.raises(asyncio.CancelledError): - await queued - finally: - release.set() - await first - finally: - marker.reset(token) - assert calls == ["caller"] - - -@pytest.mark.asyncio -async def test_markdown_renderer_is_serialized_across_components(tmp_path: Path) -> None: - """Independent chunker instances cannot overlap renderer registry changes.""" - from mistletoe.markdown_renderer import MarkdownRenderer - - path = tmp_path / "source.md" - path.write_text("# Title\n\nparagraph **bold** [[other.md]]\n") - entered, release = threading.Event(), threading.Event() - calls = [] - original_enter = MarkdownRenderer.__enter__ - - def blocking_enter(renderer: MarkdownRenderer) -> MarkdownRenderer: - calls.append(threading.get_ident()) - entered.set() - assert release.wait(5) - return original_enter(renderer) - - with patch.object(MarkdownRenderer, "__enter__", blocking_enter): - first = asyncio.create_task(MarkdownFileChunker().chunk(path)) - await _wait_started(entered) - second = asyncio.create_task(MarkdownFileChunker().chunk(path)) - try: - await asyncio.sleep(0.05) - assert len(calls) == 1 - finally: - release.set() - first_output, second_output = await asyncio.gather(first, second) - assert first_output == second_output - assert len(calls) == 2 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("outcome", ["complete", "cancel", "change", "error"]) -async def test_existing_content_searchable_during_indexing( - tmp_path: Path, - outcome: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Exercise UpdateIndexStep and LocalFileStore while parsing is suspended.""" - monkeypatch.chdir(tmp_path) - path = tmp_path / "source.txt" - path.write_text("old searchable content") - context = ApplicationContext(workspace_dir=str(tmp_path)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - store = LocalFileStore(embedding_store="") - async with store, chunker: - await store.upsert([await chunker.chunk(path)]) - path.write_text("new searchable content") - entered, release = threading.Event(), threading.Event() - original = chunker._chunk_sync - - def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - result = original(file_path, rel_path) - entered.set() - assert release.wait(5) - if outcome == "error": - raise ValueError("parser failed") - return result - - step = UpdateIndexStep(file_store=store, persist=False, app_context=context) - app = Application( - workspace_dir=str(tmp_path / "service"), - service={"backend": "http", "web_enabled": False}, - components={}, - jobs={}, - enable_logo=False, - log_to_console=False, - log_to_file=False, - ) - service = app.context.service - assert isinstance(service, HttpService) - service.build_service(app) - service.add_job(_StoreSearchJob(store)) - service.add_job(_StoreSearchStreamJob(store)) - with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): - updating = asyncio.create_task(step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}]))) - await _wait_started(entered) - try: - nodes = await store.get_nodes() - assert len(nodes) == 1 - matches = await store.keyword_search("searchable", limit=10, search_filter={}) - assert matches[0].text == "old searchable content" - async with AsyncClient(transport=ASGITransport(app=service.service), base_url="http://test") as client: - response = await client.post("/search", json={"query": "searchable"}) - assert response.status_code == 200 - assert response.json()["answer"] == ["old searchable content"] - streaming = await client.post("/search_stream", json={"query": "searchable"}) - assert streaming.status_code == 200 - assert "old searchable content" in streaming.text - assert streaming.text.endswith("data:[DONE]\n\n") - assert service.mcp_server is not None - tool = await service.mcp_server.get_tool("search") - assert tool is not None - result = await tool.run({"query": "searchable"}) - assert "old searchable content" in str(result.content) - if outcome == "cancel": - updating.cancel() - elif outcome == "change": - path.write_text("latest source content") - finally: - release.set() - if outcome == "cancel": - with pytest.raises(asyncio.CancelledError): - await updating - else: - response = await updating - assert response.success is (outcome in ("complete", "change")) - nodes = await store.get_nodes() - chunks = [store.file_chunks[chunk_id] for chunk_id in nodes[0].chunk_ids] - expected_text = { - "complete": "new searchable content", - "change": "latest source content", - "cancel": "old searchable content", - "error": "old searchable content", - }[outcome] - assert chunks[0].text == expected_text - if outcome in ("cancel", "error"): - recovered = await step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}])) - assert recovered.success is True - nodes = await store.get_nodes() - assert store.file_chunks[nodes[0].chunk_ids[0]].text == path.read_text() - assert not step._source_versions - - -@pytest.mark.asyncio -@pytest.mark.parametrize("change", ["modify", "delete", "replace", "oversize", "cancel"]) -async def test_changed_batched_file_is_rebuilt_before_publication_and_valid_files_update( - tmp_path: Path, - change: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """An earlier parsed file can change while the batch waits for the next worker.""" - monkeypatch.chdir(tmp_path) - first_path, second_path = tmp_path / "first.txt", tmp_path / "second.txt" - first_path.write_text("original first") - second_path.write_text("original second") - context = ApplicationContext(workspace_dir=str(tmp_path)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - async with LocalFileStore(embedding_store="") as store, chunker: - await store.upsert([await chunker.chunk(first_path), await chunker.chunk(second_path)]) - first_path.write_text("updated first") - second_path.write_text("updated second") - entered, release = threading.Event(), threading.Event() - original = chunker.chunk - - async def blocking_second(file_path: Path) -> tuple[FileNode, list[FileChunk]]: - if file_path == second_path: - entered.set() - while not release.is_set(): - await asyncio.sleep(0.01) - return await original(file_path) - - changes = [{"change": "modified", "path": str(path)} for path in (first_path, second_path)] - step = UpdateIndexStep(file_store=store, persist=False, app_context=context) - runtime_context = ( - RuntimeContext(changes=changes, max_file_bytes=20) - if change == "oversize" - else RuntimeContext(changes=changes) - ) - with patch.object(chunker, "chunk", side_effect=blocking_second): - task = asyncio.create_task(step(runtime_context)) - await _wait_started(entered) - try: - if change == "modify": - first_path.write_text("latest first") - elif change == "delete": - first_path.unlink() - elif change == "cancel": - task.cancel() - elif change == "oversize": - first_path.write_text("content exceeds the configured limit") - else: - replacement = tmp_path / "replacement" - replacement.write_text("replacement first") - replacement.replace(first_path) - finally: - release.set() - if change == "cancel": - with pytest.raises(asyncio.CancelledError): - await task - else: - response = await task - expected_success = change in ("modify", "replace", "oversize") - assert response.success is expected_success - assert [result["success"] for result in response.answer] == ( - [True, True] if expected_success else [False, True] - ) - if change == "oversize": - assert response.answer[0]["skipped"] is True - assert response.answer[0]["reason"] == "file_too_large" - assert not step._source_versions - nodes = {node.path: node for node in await store.get_nodes()} - if change == "oversize": - assert "first.txt" not in nodes - else: - expected_first = { - "modify": "latest first", - "delete": "original first", - "replace": "replacement first", - "cancel": "original first", - }[change] - assert store.file_chunks[nodes["first.txt"].chunk_ids[0]].text == expected_first - assert store.file_chunks[nodes["second.txt"].chunk_ids[0]].text == ( - "original second" if change == "cancel" else "updated second" - ) - recovered = await step( - RuntimeContext( - changes=[{"change": "deleted" if change == "delete" else "modified", "path": str(first_path)}], - ), - ) - assert recovered.success is True - nodes = {node.path: node for node in await store.get_nodes()} - if change == "delete": - assert "first.txt" not in nodes - else: - assert store.file_chunks[nodes["first.txt"].chunk_ids[0]].text == first_path.read_text() - assert not step._source_versions - - -@pytest.mark.asyncio -async def test_stale_batch_chunks_are_released_before_source_retry( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The retry parse does not overlap the retained stale chunk batch in memory.""" - monkeypatch.chdir(tmp_path) - source_path = tmp_path / "source.txt" - source_path.write_text("original source") - context = ApplicationContext(workspace_dir=str(tmp_path)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - async with LocalFileStore(embedding_store="") as store, chunker: - await store.upsert([await chunker.chunk(source_path)]) - source_path.write_text("parsed source") - flush_waiting = asyncio.Event() - original_guard = store._maintenance_guard - - @asynccontextmanager - async def track_store_guard(): - flush_waiting.set() - async with original_guard(): - yield - - monkeypatch.setattr(store, "_maintenance_guard", track_store_guard) - step = UpdateIndexStep(file_store=store, persist=False, batch_max_files=1, app_context=context) - original_build = step._try_build_item - stale_chunk_released: threading.Event | None = None - - async def capture_and_check_stale_chunk( - change: Change, - path: str, - abs_path: Path, - size_bytes: int, - results: list[dict], - ) -> tuple[tuple[FileNode, list[FileChunk]], int] | None: - nonlocal stale_chunk_released - if stale_chunk_released is not None: - gc.collect() - assert stale_chunk_released.is_set() - built = await original_build(change, path, abs_path, size_bytes, results) - if stale_chunk_released is None and built is not None: - stale_chunk_released = threading.Event() - weakref.finalize(built[0][1][0], stale_chunk_released.set) - return built - - monkeypatch.setattr(step, "_try_build_item", capture_and_check_stale_chunk) - await store._maintenance_lock.acquire() - try: - task = asyncio.create_task( - step( - RuntimeContext( - changes=[{"change": "modified", "path": str(source_path)}], - retry_source_changes_before_watch=True, - ), - ), - ) - await asyncio.wait_for(flush_waiting.wait(), timeout=5) - source_path.write_text("latest source with a different size") - finally: - store._maintenance_lock.release() - response = await task - - assert response.success is True - assert stale_chunk_released is not None and stale_chunk_released.is_set() - assert not step._source_versions - nodes = await store.get_nodes() - assert len(nodes) == 1 - assert store.file_chunks[nodes[0].chunk_ids[0]].text == "latest source with a different size" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("new_size", [12, 100]) -async def test_parse_retry_rechecks_size_limit_and_refreshes_memory_estimate( - tmp_path: Path, - new_size: int, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A parse retry rechecks the size cap and estimates memory from the new size.""" - monkeypatch.chdir(tmp_path) - path = tmp_path / "source.txt" - path.write_text("small") - context = ApplicationContext(workspace_dir=str(tmp_path)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - async with LocalFileStore(embedding_store="") as store, chunker: - await store.upsert([await chunker.chunk(path)]) - entered, release = threading.Event(), threading.Event() - calls: list[Path] = [] - original_chunk = chunker._chunk_sync - - def change_during_first_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: - result = original_chunk(file_path, rel_path) - calls.append(file_path) - if len(calls) == 1: - entered.set() - assert release.wait(5) - return result - - step = UpdateIndexStep(file_store=store, persist=False, app_context=context) - estimated_sizes: list[int] = [] - original_estimate = step.estimate_item_memory - - def record_estimate(item: tuple[FileNode, list[FileChunk]], file_path: Path, size_bytes: int) -> int: - estimated_sizes.append(size_bytes) - return original_estimate(item, file_path, size_bytes) - - monkeypatch.setattr(step, "estimate_item_memory", record_estimate) - with patch.object(chunker, "_chunk_sync", side_effect=change_during_first_parse): - task = asyncio.create_task( - step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}], max_file_bytes=20)), - ) - await _wait_started(entered) - path.write_text("x" * new_size) - release.set() - response = await task - - assert response.success is True - assert len(response.answer) == 1 - if new_size > 20: - assert response.answer[0]["skipped"] is True - assert response.answer[0]["reason"] == "file_too_large" - assert calls == [path] - assert not estimated_sizes - assert await store.get_nodes() == [] - else: - assert response.answer[0]["success"] is True - assert len(calls) == 2 - assert estimated_sizes == [new_size] - nodes = await store.get_nodes() - assert store.file_chunks[nodes[0].chunk_ids[0]].text == path.read_text() - assert not step._source_versions - - -@pytest.mark.asyncio -async def test_index_updates_deduplicate_absolute_and_relative_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Absolute and workspace-relative aliases of one file enter a batch once.""" - monkeypatch.chdir(tmp_path) - path = tmp_path / "source.txt" - path.write_text("source") - context = ApplicationContext(workspace_dir=str(tmp_path)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - async with LocalFileStore(embedding_store="") as store, chunker: - step = UpdateIndexStep(file_store=store, persist=False, app_context=context) - response = await step( - RuntimeContext( - changes=[ - {"change": "modified", "path": str(path)}, - {"change": "modified", "path": "source.txt"}, - ], - ), - ) - - assert response.success is True - assert response.answer == [{"change": "modified", "path": str(path), "success": True}] - nodes = await store.get_nodes() - assert len(nodes) == 1 - assert nodes[0].path == "source.txt" - assert not step._source_versions - - -@pytest.mark.asyncio -@pytest.mark.parametrize("outcome", ["change", "cancel"]) -async def test_version_is_checked_after_waiting_for_store_lock( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - outcome: str, -) -> None: - """A delayed mutation lock cannot admit a result that became stale while waiting.""" - monkeypatch.chdir(tmp_path) - path = tmp_path / "source.txt" - path.write_text("original source") - context = ApplicationContext(workspace_dir=str(tmp_path)) - chunker = DefaultFileChunker(app_context=context) - context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} - async with LocalFileStore(embedding_store="") as store, chunker: - await store.upsert([await chunker.chunk(path)]) - path.write_text("updated source") - built = asyncio.Event() - step = UpdateIndexStep(file_store=store, persist=False, app_context=context) - original = step.build_item - - async def record_build(file_path: Path) -> tuple[FileNode, list[FileChunk]]: - item = await original(file_path) - built.set() - return item - - with patch.object(step, "build_item", side_effect=record_build): - async with store._maintenance_guard(): - task = asyncio.create_task(step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}]))) - await asyncio.wait_for(built.wait(), 5) - await asyncio.sleep(0) - if outcome == "cancel": - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - else: - path.write_text("latest source") - if outcome == "change": - response = await task - assert response.success is True - assert not step._source_versions - nodes = await store.get_nodes() - expected_text = "latest source" if outcome == "change" else "original source" - assert store.file_chunks[nodes[0].chunk_ids[0]].text == expected_text - if outcome == "cancel": - recovered = await step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}])) - assert recovered.success is True - nodes = await store.get_nodes() - assert store.file_chunks[nodes[0].chunk_ids[0]].text == path.read_text() - assert not step._source_versions + ) + assert response.success is True + assert await store.get_nodes() == [] + assert not await store.keyword_search("unique", limit=10, search_filter={}) + + +@pytest.mark.asyncio +async def test_shutdown_rejects_queued_work_and_drains_despite_cancellation(tmp_path: Path) -> None: + """Closing waits for running work, rejects queued parsing, and permits restart.""" + path = tmp_path / "source.txt" + path.write_text("source") + chunker = DefaultFileChunker() + entered, release = threading.Event(), threading.Event() + original = chunker._chunk_sync + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + entered.set() + assert release.wait(5) + return original(file_path, rel_path) + + await chunker.start() + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + running = asyncio.create_task(chunker.chunk(path)) + await _wait_started(entered) + queued = asyncio.create_task(chunker.chunk(path)) + await asyncio.sleep(0) + closing = asyncio.create_task(chunker.close()) + await asyncio.sleep(0.01) + closing.cancel() + try: + assert not closing.done() + finally: + release.set() + with pytest.raises(RuntimeError, match="closed"): + await running + with pytest.raises(RuntimeError, match="closed"): + await queued + with pytest.raises(asyncio.CancelledError): + await closing + assert chunker._worker is None + assert not chunker.is_started + async with chunker: + assert (await chunker.chunk(path))[1][0].text == "source" + + +@pytest.mark.asyncio +async def test_shutdown_from_another_event_loop_waits_for_worker_and_closes_executor(tmp_path: Path) -> None: + """Worker admission and shutdown can cross event loops without stranding the executor.""" + path = tmp_path / "source.txt" + path.write_text("source") + chunker = DefaultFileChunker() + entered, release = threading.Event(), threading.Event() + worker_errors: list[BaseException] = [] + original = chunker._chunk_sync + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + entered.set() + assert release.wait(5) + return original(file_path, rel_path) + + def run_on_worker_loop() -> None: + try: + asyncio.run(chunker.chunk(path)) + except BaseException as exc: + worker_errors.append(exc) + + await chunker.start() + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + worker_thread = threading.Thread(target=run_on_worker_loop) + worker_thread.start() + await _wait_started(entered) + closing = asyncio.create_task(chunker.close()) + try: + for _ in range(100): + if chunker._worker_closing: + break + await asyncio.sleep(0.01) + assert chunker._worker_closing + assert not closing.done() + finally: + release.set() + await closing + await asyncio.to_thread(worker_thread.join, 5) + + assert not worker_thread.is_alive() + assert len(worker_errors) == 1 + assert isinstance(worker_errors[0], RuntimeError) + assert "different event loop" not in str(worker_errors[0]) + assert chunker._worker is None + assert not chunker.is_started + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["modify", "replace", "delete"]) +async def test_source_changes_do_not_return_stale_chunks(tmp_path: Path, change: str) -> None: + """Changes after reading cannot be mistaken for successfully parsed current content.""" + path = tmp_path / "source.txt" + path.write_text("old source") + chunker = DefaultFileChunker() + original = chunker._chunk_sync + entered, release = threading.Event(), threading.Event() + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + result = original(file_path, rel_path) + entered.set() + assert release.wait(5) + return result + + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + task = asyncio.create_task(chunker.chunk(path)) + await _wait_started(entered) + try: + if change == "modify": + path.write_text("new source with different size") + elif change == "replace": + replacement = tmp_path / "replacement" + replacement.write_text("replacement source") + replacement.replace(path) + else: + path.unlink() + finally: + release.set() + with pytest.raises((RuntimeError, FileNotFoundError)): + await task + + +@pytest.mark.asyncio +async def test_change_before_event_loop_receives_result_is_rejected(tmp_path: Path) -> None: + """Validate the worker snapshot again when its result reaches the calling loop.""" + path = tmp_path / "source.txt" + path.write_text("old source") + chunker = DefaultFileChunker() + original = chunker._chunk_unchanged_file + loop = asyncio.get_running_loop() + changed = threading.Event() + + def change_on_loop() -> None: + path.write_text("changed after the worker snapshot") + changed.set() + + def change_after_parse( + file_path: Path, + rel_path: str, + chunk: Callable[[Path, str], tuple[FileNode, list[FileChunk]]], + ) -> tuple[tuple[FileNode, list[FileChunk]], os.stat_result]: + result = original(file_path, rel_path, chunk) + loop.call_soon_threadsafe(change_on_loop) + assert changed.wait(5) + return result + + with patch.object(chunker, "_chunk_unchanged_file", side_effect=change_after_parse): + with pytest.raises(RuntimeError, match="File changed"): + await chunker.chunk(path) + + +@pytest.mark.asyncio +async def test_queued_cancellation_does_not_submit_and_context_is_preserved(tmp_path: Path) -> None: + """A cancelled waiter uses no worker slot; caller context is copied into the worker.""" + path = tmp_path / "source.txt" + path.write_text("source") + chunker = DefaultFileChunker() + entered, release = threading.Event(), threading.Event() + marker = contextvars.ContextVar("chunk_test_marker", default="unset") + original = chunker._chunk_sync + calls = [] + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + calls.append(marker.get()) + entered.set() + assert release.wait(5) + return original(file_path, rel_path) + + token = marker.set("caller") + try: + async with chunker: + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + first = asyncio.create_task(chunker.chunk(path)) + await _wait_started(entered) + queued = asyncio.create_task(chunker.chunk(path)) + await asyncio.sleep(0) + queued.cancel() + try: + with pytest.raises(asyncio.CancelledError): + await queued + finally: + release.set() + await first + finally: + marker.reset(token) + assert calls == ["caller"] + + +@pytest.mark.asyncio +async def test_markdown_renderer_is_serialized_across_components(tmp_path: Path) -> None: + """Independent chunker instances cannot overlap renderer registry changes.""" + from mistletoe.markdown_renderer import MarkdownRenderer + + path = tmp_path / "source.md" + path.write_text("# Title\n\nparagraph **bold** [[other.md]]\n") + entered, release = threading.Event(), threading.Event() + calls = [] + original_enter = MarkdownRenderer.__enter__ + + def blocking_enter(renderer: MarkdownRenderer) -> MarkdownRenderer: + calls.append(threading.get_ident()) + entered.set() + assert release.wait(5) + return original_enter(renderer) + + with patch.object(MarkdownRenderer, "__enter__", blocking_enter): + first = asyncio.create_task(MarkdownFileChunker().chunk(path)) + await _wait_started(entered) + second = asyncio.create_task(MarkdownFileChunker().chunk(path)) + try: + await asyncio.sleep(0.05) + assert len(calls) == 1 + finally: + release.set() + first_output, second_output = await asyncio.gather(first, second) + assert first_output == second_output + assert len(calls) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["complete", "cancel", "change", "error"]) +async def test_existing_content_searchable_during_indexing( + tmp_path: Path, + outcome: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Exercise UpdateIndexStep and LocalFileStore while parsing is suspended.""" + monkeypatch.chdir(tmp_path) + path = tmp_path / "source.txt" + path.write_text("old searchable content") + context = ApplicationContext(workspace_dir=str(tmp_path)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + store = LocalFileStore(embedding_store="") + async with store, chunker: + await store.upsert([await chunker.chunk(path)]) + path.write_text("new searchable content") + entered, release = threading.Event(), threading.Event() + original = chunker._chunk_sync + + def blocking_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + result = original(file_path, rel_path) + entered.set() + assert release.wait(5) + if outcome == "error": + raise ValueError("parser failed") + return result + + step = UpdateIndexStep(file_store=store, persist=False, app_context=context) + app = Application( + workspace_dir=str(tmp_path / "service"), + service={"backend": "http", "web_enabled": False}, + components={}, + jobs={}, + enable_logo=False, + log_to_console=False, + log_to_file=False, + ) + service = app.context.service + assert isinstance(service, HttpService) + service.build_service(app) + service.add_job(_StoreSearchJob(store)) + service.add_job(_StoreSearchStreamJob(store)) + with patch.object(chunker, "_chunk_sync", side_effect=blocking_parse): + updating = asyncio.create_task(step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}]))) + await _wait_started(entered) + try: + nodes = await store.get_nodes() + assert len(nodes) == 1 + matches = await store.keyword_search("searchable", limit=10, search_filter={}) + assert matches[0].text == "old searchable content" + async with AsyncClient(transport=ASGITransport(app=service.service), base_url="http://test") as client: + response = await client.post("/search", json={"query": "searchable"}) + assert response.status_code == 200 + assert response.json()["answer"] == ["old searchable content"] + streaming = await client.post("/search_stream", json={"query": "searchable"}) + assert streaming.status_code == 200 + assert "old searchable content" in streaming.text + assert streaming.text.endswith("data:[DONE]\n\n") + assert service.mcp_server is not None + tool = await service.mcp_server.get_tool("search") + assert tool is not None + result = await tool.run({"query": "searchable"}) + assert "old searchable content" in str(result.content) + if outcome == "cancel": + updating.cancel() + elif outcome == "change": + path.write_text("latest source content") + finally: + release.set() + if outcome == "cancel": + with pytest.raises(asyncio.CancelledError): + await updating + else: + response = await updating + assert response.success is (outcome in ("complete", "change")) + nodes = await store.get_nodes() + chunks = [store.file_chunks[chunk_id] for chunk_id in nodes[0].chunk_ids] + expected_text = { + "complete": "new searchable content", + "change": "latest source content", + "cancel": "old searchable content", + "error": "old searchable content", + }[outcome] + assert chunks[0].text == expected_text + if outcome in ("cancel", "error"): + recovered = await step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}])) + assert recovered.success is True + nodes = await store.get_nodes() + assert store.file_chunks[nodes[0].chunk_ids[0]].text == path.read_text() + assert not step._source_versions + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["modify", "delete", "replace", "oversize", "cancel"]) +async def test_changed_batched_file_is_rebuilt_before_publication_and_valid_files_update( + tmp_path: Path, + change: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An earlier parsed file can change while the batch waits for the next worker.""" + monkeypatch.chdir(tmp_path) + first_path, second_path = tmp_path / "first.txt", tmp_path / "second.txt" + first_path.write_text("original first") + second_path.write_text("original second") + context = ApplicationContext(workspace_dir=str(tmp_path)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + async with LocalFileStore(embedding_store="") as store, chunker: + await store.upsert([await chunker.chunk(first_path), await chunker.chunk(second_path)]) + first_path.write_text("updated first") + second_path.write_text("updated second") + entered, release = threading.Event(), threading.Event() + original = chunker.chunk + + async def blocking_second(file_path: Path) -> tuple[FileNode, list[FileChunk]]: + if file_path == second_path: + entered.set() + while not release.is_set(): + await asyncio.sleep(0.01) + return await original(file_path) + + changes = [{"change": "modified", "path": str(path)} for path in (first_path, second_path)] + step = UpdateIndexStep(file_store=store, persist=False, app_context=context) + runtime_context = ( + RuntimeContext(changes=changes, max_file_bytes=20) + if change == "oversize" + else RuntimeContext(changes=changes) + ) + with patch.object(chunker, "chunk", side_effect=blocking_second): + task = asyncio.create_task(step(runtime_context)) + await _wait_started(entered) + try: + if change == "modify": + first_path.write_text("latest first") + elif change == "delete": + first_path.unlink() + elif change == "cancel": + task.cancel() + elif change == "oversize": + first_path.write_text("content exceeds the configured limit") + else: + replacement = tmp_path / "replacement" + replacement.write_text("replacement first") + replacement.replace(first_path) + finally: + release.set() + if change == "cancel": + with pytest.raises(asyncio.CancelledError): + await task + else: + response = await task + expected_success = change in ("modify", "replace", "oversize") + assert response.success is expected_success + assert [result["success"] for result in response.answer] == ( + [True, True] if expected_success else [False, True] + ) + if change == "oversize": + assert response.answer[0]["skipped"] is True + assert response.answer[0]["reason"] == "file_too_large" + assert not step._source_versions + nodes = {node.path: node for node in await store.get_nodes()} + if change == "oversize": + assert "first.txt" not in nodes + else: + expected_first = { + "modify": "latest first", + "delete": "original first", + "replace": "replacement first", + "cancel": "original first", + }[change] + assert store.file_chunks[nodes["first.txt"].chunk_ids[0]].text == expected_first + assert store.file_chunks[nodes["second.txt"].chunk_ids[0]].text == ( + "original second" if change == "cancel" else "updated second" + ) + recovered = await step( + RuntimeContext( + changes=[{"change": "deleted" if change == "delete" else "modified", "path": str(first_path)}], + ), + ) + assert recovered.success is True + nodes = {node.path: node for node in await store.get_nodes()} + if change == "delete": + assert "first.txt" not in nodes + else: + assert store.file_chunks[nodes["first.txt"].chunk_ids[0]].text == first_path.read_text() + assert not step._source_versions + + +@pytest.mark.asyncio +async def test_stale_batch_chunks_are_released_before_source_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The retry parse does not overlap the retained stale chunk batch in memory.""" + monkeypatch.chdir(tmp_path) + source_path = tmp_path / "source.txt" + source_path.write_text("original source") + context = ApplicationContext(workspace_dir=str(tmp_path)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + async with LocalFileStore(embedding_store="") as store, chunker: + await store.upsert([await chunker.chunk(source_path)]) + source_path.write_text("parsed source") + flush_waiting = asyncio.Event() + original_guard = store._maintenance_guard + + @asynccontextmanager + async def track_store_guard(): + flush_waiting.set() + async with original_guard(): + yield + + monkeypatch.setattr(store, "_maintenance_guard", track_store_guard) + step = UpdateIndexStep(file_store=store, persist=False, batch_max_files=1, app_context=context) + original_build = step._try_build_item + stale_chunk_released: threading.Event | None = None + + async def capture_and_check_stale_chunk( + change: Change, + path: str, + abs_path: Path, + size_bytes: int, + results: list[dict], + ) -> tuple[tuple[FileNode, list[FileChunk]], int] | None: + nonlocal stale_chunk_released + if stale_chunk_released is not None: + gc.collect() + assert stale_chunk_released.is_set() + built = await original_build(change, path, abs_path, size_bytes, results) + if stale_chunk_released is None and built is not None: + stale_chunk_released = threading.Event() + weakref.finalize(built[0][1][0], stale_chunk_released.set) + return built + + monkeypatch.setattr(step, "_try_build_item", capture_and_check_stale_chunk) + await store._maintenance_lock.acquire() + try: + task = asyncio.create_task( + step( + RuntimeContext( + changes=[{"change": "modified", "path": str(source_path)}], + retry_source_changes_before_watch=True, + ), + ), + ) + await asyncio.wait_for(flush_waiting.wait(), timeout=5) + source_path.write_text("latest source with a different size") + finally: + store._maintenance_lock.release() + response = await task + + assert response.success is True + assert stale_chunk_released is not None and stale_chunk_released.is_set() + assert not step._source_versions + nodes = await store.get_nodes() + assert len(nodes) == 1 + assert store.file_chunks[nodes[0].chunk_ids[0]].text == "latest source with a different size" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("new_size", [12, 100]) +async def test_parse_retry_rechecks_size_limit_and_refreshes_memory_estimate( + tmp_path: Path, + new_size: int, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A parse retry rechecks the size cap and estimates memory from the new size.""" + monkeypatch.chdir(tmp_path) + path = tmp_path / "source.txt" + path.write_text("small") + context = ApplicationContext(workspace_dir=str(tmp_path)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + async with LocalFileStore(embedding_store="") as store, chunker: + await store.upsert([await chunker.chunk(path)]) + entered, release = threading.Event(), threading.Event() + calls: list[Path] = [] + original_chunk = chunker._chunk_sync + + def change_during_first_parse(file_path: Path, rel_path: str) -> tuple[FileNode, list[FileChunk]]: + result = original_chunk(file_path, rel_path) + calls.append(file_path) + if len(calls) == 1: + entered.set() + assert release.wait(5) + return result + + step = UpdateIndexStep(file_store=store, persist=False, app_context=context) + estimated_sizes: list[int] = [] + original_estimate = step.estimate_item_memory + + def record_estimate(item: tuple[FileNode, list[FileChunk]], file_path: Path, size_bytes: int) -> int: + estimated_sizes.append(size_bytes) + return original_estimate(item, file_path, size_bytes) + + monkeypatch.setattr(step, "estimate_item_memory", record_estimate) + with patch.object(chunker, "_chunk_sync", side_effect=change_during_first_parse): + task = asyncio.create_task( + step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}], max_file_bytes=20)), + ) + await _wait_started(entered) + path.write_text("x" * new_size) + release.set() + response = await task + + assert response.success is True + assert len(response.answer) == 1 + if new_size > 20: + assert response.answer[0]["skipped"] is True + assert response.answer[0]["reason"] == "file_too_large" + assert calls == [path] + assert not estimated_sizes + assert await store.get_nodes() == [] + else: + assert response.answer[0]["success"] is True + assert len(calls) == 2 + assert estimated_sizes == [new_size] + nodes = await store.get_nodes() + assert store.file_chunks[nodes[0].chunk_ids[0]].text == path.read_text() + assert not step._source_versions + + +@pytest.mark.asyncio +async def test_index_updates_deduplicate_absolute_and_relative_paths( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Absolute and workspace-relative aliases of one file enter a batch once.""" + monkeypatch.chdir(tmp_path) + path = tmp_path / "source.txt" + path.write_text("source") + context = ApplicationContext(workspace_dir=str(tmp_path)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + async with LocalFileStore(embedding_store="") as store, chunker: + step = UpdateIndexStep(file_store=store, persist=False, app_context=context) + response = await step( + RuntimeContext( + changes=[ + {"change": "modified", "path": str(path)}, + {"change": "modified", "path": "source.txt"}, + ], + ), + ) + + assert response.success is True + assert response.answer == [{"change": "modified", "path": str(path), "success": True}] + nodes = await store.get_nodes() + assert len(nodes) == 1 + assert nodes[0].path == "source.txt" + assert not step._source_versions + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["change", "cancel"]) +async def test_version_is_checked_after_waiting_for_store_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + outcome: str, +) -> None: + """A delayed mutation lock cannot admit a result that became stale while waiting.""" + monkeypatch.chdir(tmp_path) + path = tmp_path / "source.txt" + path.write_text("original source") + context = ApplicationContext(workspace_dir=str(tmp_path)) + chunker = DefaultFileChunker(app_context=context) + context.components = {ComponentEnum.FILE_CHUNKER: {"default": chunker}} + async with LocalFileStore(embedding_store="") as store, chunker: + await store.upsert([await chunker.chunk(path)]) + path.write_text("updated source") + built = asyncio.Event() + step = UpdateIndexStep(file_store=store, persist=False, app_context=context) + original = step.build_item + + async def record_build(file_path: Path) -> tuple[FileNode, list[FileChunk]]: + item = await original(file_path) + built.set() + return item + + with patch.object(step, "build_item", side_effect=record_build): + async with store._maintenance_guard(): + task = asyncio.create_task(step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}]))) + await asyncio.wait_for(built.wait(), 5) + await asyncio.sleep(0) + if outcome == "cancel": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + path.write_text("latest source") + if outcome == "change": + response = await task + assert response.success is True + assert not step._source_versions + nodes = await store.get_nodes() + expected_text = "latest source" if outcome == "change" else "original source" + assert store.file_chunks[nodes[0].chunk_ids[0]].text == expected_text + if outcome == "cancel": + recovered = await step(RuntimeContext(changes=[{"change": "modified", "path": str(path)}])) + assert recovered.success is True + nodes = await store.get_nodes() + assert store.file_chunks[nodes[0].chunk_ids[0]].text == path.read_text() + assert not step._source_versions