From 1ec6536b32464e778d9d1947428e31a10fc279fa Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Tue, 8 Sep 2026 01:14:37 +0800 Subject: [PATCH] fix(index): invalidate chunks when chunkers change --- .../file_chunker/base_file_chunker.py | 16 +++++ .../file_chunker/default_file_chunker.py | 12 ++++ .../file_chunker/json_file_chunker.py | 11 +++ .../file_chunker/jsonl_file_chunker.py | 13 ++++ .../file_chunker/markdown_file_chunker.py | 14 ++++ .../file_store/faiss_local_file_store.py | 10 ++- .../components/file_store/local_file_store.py | 31 +++++++- .../file_store/zvec_local_file_store.py | 10 ++- tests/unit/test_chunk_store_fingerprint.py | 71 +++++++++++++++++++ 9 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_chunk_store_fingerprint.py diff --git a/reme/components/file_chunker/base_file_chunker.py b/reme/components/file_chunker/base_file_chunker.py index 49312989..747639f0 100644 --- a/reme/components/file_chunker/base_file_chunker.py +++ b/reme/components/file_chunker/base_file_chunker.py @@ -1,6 +1,8 @@ """Abstract base for file chunkers.""" from abc import abstractmethod +import hashlib +import json from pathlib import Path from ..base_component import BaseComponent @@ -17,6 +19,20 @@ class BaseFileChunker(BaseComponent): super().__init__(**kwargs) self.supported_extensions: list[str] = supported_extensions or [] + def config_fingerprint(self) -> str: + """Return a stable fingerprint for the chunker's derived output shape.""" + payload = self._config_fingerprint_payload() + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] + + def _config_fingerprint_payload(self) -> dict: + """Return the structured state that should invalidate persisted chunks.""" + return { + "type": type(self).__name__, + "backend": self.backend, + "supported_extensions": list(self.supported_extensions), + } + @abstractmethod async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: """Chunk a file into (node, chunks).""" diff --git a/reme/components/file_chunker/default_file_chunker.py b/reme/components/file_chunker/default_file_chunker.py index 4fddc632..d71326a9 100644 --- a/reme/components/file_chunker/default_file_chunker.py +++ b/reme/components/file_chunker/default_file_chunker.py @@ -38,6 +38,18 @@ class DefaultFileChunker(BaseFileChunker): self.chunk_byte_size = max(100, chunk_byte_size) self.overlap_byte_size = max(4, overlap_byte_size) + def _config_fingerprint_payload(self) -> dict: + payload = super()._config_fingerprint_payload() + payload.update( + { + "encoding": self.encoding, + "invalid_encoding_policy": self.invalid_encoding_policy, + "chunk_byte_size": self.chunk_byte_size, + "overlap_byte_size": self.overlap_byte_size, + }, + ) + return payload + @staticmethod def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str]: """Parse YAML front matter delimited by ---, return (front_matter, remaining).""" diff --git a/reme/components/file_chunker/json_file_chunker.py b/reme/components/file_chunker/json_file_chunker.py index e8cce166..1357b55c 100644 --- a/reme/components/file_chunker/json_file_chunker.py +++ b/reme/components/file_chunker/json_file_chunker.py @@ -34,6 +34,17 @@ class JsonFileChunker(BaseFileChunker): self.chunk_chars = max(256, chunk_chars) self.min_element_size = max(64, int(self.chunk_chars * 0.05)) + def _config_fingerprint_payload(self) -> dict: + payload = super()._config_fingerprint_payload() + payload.update( + { + "encoding": self.encoding, + "chunk_chars": self.chunk_chars, + "min_element_size": self.min_element_size, + }, + ) + return payload + class Node: """AST node for JSON tree construction.""" diff --git a/reme/components/file_chunker/jsonl_file_chunker.py b/reme/components/file_chunker/jsonl_file_chunker.py index 9417282c..e9d20ffd 100644 --- a/reme/components/file_chunker/jsonl_file_chunker.py +++ b/reme/components/file_chunker/jsonl_file_chunker.py @@ -46,6 +46,19 @@ class JsonlFileChunker(BaseFileChunker): self.max_lines_per_chunk = max(1, max_lines_per_chunk) if max_lines_per_chunk is not None else None self.mode = mode + def _config_fingerprint_payload(self) -> dict: + payload = super()._config_fingerprint_payload() + payload.update( + { + "encoding": self.encoding, + "max_chars": self.max_chars, + "max_overlap_chars": self.max_overlap_chars, + "max_lines_per_chunk": self.max_lines_per_chunk, + "mode": self.mode, + }, + ) + return payload + # ------------------------------------------------------------------ # Size helpers # ------------------------------------------------------------------ diff --git a/reme/components/file_chunker/markdown_file_chunker.py b/reme/components/file_chunker/markdown_file_chunker.py index 06a07793..7d0193c6 100644 --- a/reme/components/file_chunker/markdown_file_chunker.py +++ b/reme/components/file_chunker/markdown_file_chunker.py @@ -123,6 +123,20 @@ class MarkdownFileChunker(DefaultFileChunker): self.include_frontmatter_in_metadata = include_frontmatter_in_metadata self.include_frontmatter_keys_in_metadata = list(include_frontmatter_keys_in_metadata or []) + def _config_fingerprint_payload(self) -> dict: + payload = super()._config_fingerprint_payload() + payload.update( + { + "encoding": self.encoding, + "chunk_byte_size": self.chunk_byte_size, + "embed_toc": self.embed_toc, + "max_ast_sections": self.max_ast_sections, + "include_frontmatter_in_metadata": self.include_frontmatter_in_metadata, + "include_frontmatter_keys_in_metadata": list(self.include_frontmatter_keys_in_metadata), + }, + ) + return payload + async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: file_path = Path(path) rel_path = self.to_workspace_relative(path) diff --git a/reme/components/file_store/faiss_local_file_store.py b/reme/components/file_store/faiss_local_file_store.py index 12684e81..74c58310 100644 --- a/reme/components/file_store/faiss_local_file_store.py +++ b/reme/components/file_store/faiss_local_file_store.py @@ -83,8 +83,6 @@ class FaissLocalFileStore(LocalFileStore): self.hnsw_m = hnsw_m self.hnsw_ef_construction = hnsw_ef_construction self.async_reindex = async_reindex - self.faiss_path = self.component_metadata_path / f"faiss_index_{self.name}_{self.store_version}.bin" - self.faiss_idmap_path = self.component_metadata_path / f"faiss_idmap_{self.name}_{self.store_version}.json" self._faiss_index = None # faiss.Index | None self._id_map: list[str] = [] # row -> chunk_id self._id_to_row: dict[str, int] = {} # chunk_id -> row (live entries only) @@ -100,6 +98,14 @@ class FaissLocalFileStore(LocalFileStore): self._index_writes = 0 # bumped on every index mutation; used to re-arm the flag self._closing = False # set during _close() to stop spawning background reindexes + @property + def faiss_path(self): + return self.component_metadata_path / f"faiss_index_{self.name}_{self._store_version_suffix()}.bin" + + @property + def faiss_idmap_path(self): + return self.component_metadata_path / f"faiss_idmap_{self.name}_{self._store_version_suffix()}.json" + @staticmethod def _import_faiss(): try: diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index 6075aaee..9cda87e9 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -3,11 +3,13 @@ import asyncio import base64 import datetime +import hashlib import heapq import json import time from collections.abc import Iterable from contextlib import suppress +from pathlib import Path import numpy as np @@ -17,7 +19,7 @@ from ..embedding_store import BaseEmbeddingStore from ..file_graph import BaseFileGraph from ..keyword_index import BaseKeywordIndex from ..tag_index import BaseTagIndex -from ...enumeration import LinkScopeEnum +from ...enumeration import ComponentEnum, LinkScopeEnum from ...schema import FileChunk, FileLink, FileNode from ...utils import batch_cosine_similarity from ...utils.async_utils import complete_in_thread @@ -71,7 +73,6 @@ class LocalFileStore(BaseFileStore): self.encoding = encoding self.store_version = store_version self.file_chunks: dict[str, FileChunk] = {} - self.chunks_path = self.component_metadata_path / f"file_chunks_{self.name}_{self.store_version}.jsonl.zst" self._embedding_backfill_task: asyncio.Task | None = None self._embedding_backfill_pending = False self._embedding_rebuild_pending = bool(embedding_rebuild_required) @@ -81,6 +82,32 @@ class LocalFileStore(BaseFileStore): self._tag_indexed_file_count = 0 self._closing = False + def _store_version_suffix(self) -> str: + """Return the on-disk version suffix for the active chunker configuration.""" + fingerprint = self._chunker_fingerprint() + return f"{self.store_version}_{fingerprint}" if fingerprint else self.store_version + + def _chunker_fingerprint(self) -> str: + """Fingerprint the bound chunker configuration so derived chunks rotate on change.""" + if self.app_context is None: + return "" + chunkers = self.app_context.components.get(ComponentEnum.FILE_CHUNKER, {}) + if not chunkers: + return "" + payload = { + name: chunker.config_fingerprint() + for name, chunker in sorted(chunkers.items()) + if hasattr(chunker, "config_fingerprint") + } + if not payload: + return "" + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] + + @property + def chunks_path(self) -> Path: + return self.component_metadata_path / f"file_chunks_{self.name}_{self._store_version_suffix()}.jsonl.zst" + # -- lifecycle ------------------------------------------------------------ async def _start(self) -> None: diff --git a/reme/components/file_store/zvec_local_file_store.py b/reme/components/file_store/zvec_local_file_store.py index afad4b4d..d50f7017 100644 --- a/reme/components/file_store/zvec_local_file_store.py +++ b/reme/components/file_store/zvec_local_file_store.py @@ -64,11 +64,17 @@ class ZvecLocalFileStore(LocalFileStore): self._zvec = self._import_zvec() self.hnsw_m = hnsw_m self.hnsw_ef_construction = hnsw_ef_construction - self.zvec_path = self.component_metadata_path / f"zvec_index_{self.name}_{self.store_version}" - self.zvec_sidecar_path = self.component_metadata_path / f"zvec_sidecar_{self.name}_{self.store_version}.json" self._collection = None # zvec.Collection | None self._indexed_ids: set[str] = set() # chunk ids currently present in the collection + @property + def zvec_path(self): + return self.component_metadata_path / f"zvec_index_{self.name}_{self._store_version_suffix()}" + + @property + def zvec_sidecar_path(self): + return self.component_metadata_path / f"zvec_sidecar_{self.name}_{self._store_version_suffix()}.json" + @staticmethod def _import_zvec(): try: diff --git a/tests/unit/test_chunk_store_fingerprint.py b/tests/unit/test_chunk_store_fingerprint.py new file mode 100644 index 00000000..ab3788c9 --- /dev/null +++ b/tests/unit/test_chunk_store_fingerprint.py @@ -0,0 +1,71 @@ +"""Regression tests for chunker-sensitive file-store persistence.""" + +import asyncio + +from reme.components.application_context import ApplicationContext +from reme.components.file_chunker import MarkdownFileChunker +from reme.components.file_store import LocalFileStore +from reme.enumeration import ComponentEnum +from reme.schema import FileChunk, FileNode +from reme.utils.jsonl_zst import write_jsonl_zst + + +def _store_for_chunker(workspace, *, chunk_byte_size: int) -> LocalFileStore: + context = ApplicationContext(workspace_dir=str(workspace)) + chunker = MarkdownFileChunker(chunk_byte_size=chunk_byte_size, app_context=context) + context.components[ComponentEnum.FILE_CHUNKER] = {"default": chunker} + return LocalFileStore( + name="default", + embedding_store="", + app_context=context, + ) + + +def test_chunk_store_path_changes_when_chunker_configuration_changes(tmp_path): + first = _store_for_chunker(tmp_path, chunk_byte_size=1000) + second = _store_for_chunker(tmp_path, chunk_byte_size=2000) + + assert first.chunks_path != second.chunks_path + assert first.chunks_path.name.startswith("file_chunks_default_v1_") + assert second.chunks_path.name.startswith("file_chunks_default_v1_") + + +def test_chunk_store_path_is_stable_for_equivalent_chunker_configuration(tmp_path): + first = _store_for_chunker(tmp_path, chunk_byte_size=1000) + second = _store_for_chunker(tmp_path, chunk_byte_size=1000) + + assert first.chunks_path == second.chunks_path + + +def test_changed_chunker_does_not_load_legacy_chunks_and_clears_stale_graph(tmp_path): + class Graph: + def __init__(self): + self.nodes = [FileNode(path="note.md", st_mtime=1, chunk_ids=["legacy"])] + self.cleared = False + + async def get_nodes(self): + return self.nodes + + async def clear(self): + self.cleared = True + self.nodes = [] + + async def run(): + store = _store_for_chunker(tmp_path, chunk_byte_size=2000) + legacy_path = store.component_metadata_path / "file_chunks_default_v1.jsonl.zst" + legacy_path.parent.mkdir(parents=True, exist_ok=True) + write_jsonl_zst(legacy_path, [FileChunk(id="legacy", path="note.md", text="old chunk").model_dump_json()]) + + graph = Graph() + store.file_graph = graph + store.keyword_index = None + store.tag_index = None + + assert legacy_path != store.chunks_path + await store.load() + + assert store.file_chunks == {} + assert graph.cleared is True + assert legacy_path.exists() + + asyncio.run(run())