From a3bd81bde25b87609bfa29cded4b85751bde85ac Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:01:07 +0800 Subject: [PATCH] Update version to 0.4.0.2 and improve tokenizer index handling (#290) * fix(core): update version number to 0.4.0.1 - Incremented version from 0.4.0.0 to 0.4.0.1 in __init__.py * fix(index): remove stopwords path from tokenizer config and add keyword index repair - Remove stopwords_path from tokenizer config to prevent index forking by install path - Add _sync_keyword_index_from_chunks method to repair keyword index when persisted state mismatches - Implement test for keyword index repair from persisted chunks when missing - Add test to verify tokenizer fingerprint ignores stopwords absolute path - Update version from 0.4.0.1 to 0.4.0.2 * feat(dream): add scan_days parameter to dream extraction process - Add scan_days configuration option to default.yaml with default value of 2 - Implement recent_dates utility function to calculate date ranges for scanning - Modify DreamExtractStep to scan multiple days based on scan_days parameter - Update dream extraction to process files across multiple dates instead of single day - Extend DreamState schema to include dates and scan_days fields - Update DreamTopicsStep to handle multi-day topic processing - Modify finish step to checkpoint files from all scanned dates - Add comprehensive tests for multi-day scanning functionality - Update prompt templates to include scan dates information - Refactor topics writing logic to target specific date rather than current date --- reme/__init__.py | 2 +- .../components/file_store/local_file_store.py | 27 ++++++ reme/components/keyword_index/bm25_index.py | 1 - reme/config/default.yaml | 6 ++ reme/steps/evolve/dream/extract.py | 37 ++++++-- reme/steps/evolve/dream/extract.yaml | 4 + reme/steps/evolve/dream/finish.py | 10 +- reme/steps/evolve/dream/schema.py | 3 + reme/steps/evolve/dream/topics.py | 90 +++++++++++++----- reme/steps/evolve/dream/utils.py | 10 ++ tests/unit/test_auto_dream.py | 94 ++++++++++++++++++- tests/unit/test_file_store_consistency.py | 30 ++++++ tests/unit/test_keyword_index.py | 41 ++++++++ 13 files changed, 315 insertions(+), 40 deletions(-) diff --git a/reme/__init__.py b/reme/__init__.py index f5797839..90b16ac3 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -1,6 +1,6 @@ """ReMe CLI package.""" -__version__ = "0.4.0.1" +__version__ = "0.4.0.2" from . import config from . import constants diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index f8fa12f3..86d94177 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -1,5 +1,7 @@ """In-memory file store with compressed JSONL persistence on close.""" +from contextlib import suppress + import numpy as np from .base_file_store import BaseFileStore @@ -88,9 +90,34 @@ class LocalFileStore(BaseFileStore): chunk = FileChunk.model_validate_json(line) self.file_chunks[chunk.id] = chunk self.logger.info(f"Loaded {len(self.file_chunks)} chunks from {self.chunks_path}") + await self._sync_keyword_index_from_chunks() except Exception as e: self.logger.exception(f"Failed to load {self.chunks_path}: {e}") + async def _sync_keyword_index_from_chunks(self) -> None: + """Repair keyword index when its persisted state does not match chunks.""" + if not self.keyword_index or not self.file_chunks: + return + + docs = {cid: chunk.text for cid, chunk in self.file_chunks.items() if chunk.text} + if not docs: + return + + expected_ids = set(docs) + live_ids = None + with suppress(Exception): + live_ids = set(getattr(self.keyword_index, "doc_meta", {}).keys()) + + if live_ids == expected_ids: + return + + n_docs = getattr(self.keyword_index, "n_docs", None) + if live_ids is None and n_docs == len(expected_ids): + return + + self.logger.warning(f"{self.name}: keyword index mismatch with chunks; rebuilding {len(docs)} docs") + await self.keyword_index.reset_index(docs) + async def dump(self) -> None: """Atomically rewrite the JSONL, then cascade dump into keyword_index and file_graph.""" assert self.file_graph is not None diff --git a/reme/components/keyword_index/bm25_index.py b/reme/components/keyword_index/bm25_index.py index 027ec2ab..53a3c618 100644 --- a/reme/components/keyword_index/bm25_index.py +++ b/reme/components/keyword_index/bm25_index.py @@ -82,7 +82,6 @@ class BM25Index(BaseKeywordIndex): stopwords_path = getattr(self.tokenizer, "stopwords_path", None) if stopwords_path is not None: path = Path(stopwords_path) - config["stopwords_path"] = str(path) if path.exists() and path.is_file(): config["stopwords_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() else: diff --git a/reme/config/default.yaml b/reme/config/default.yaml index cab1c034..d5cacb56 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -59,6 +59,7 @@ jobs: - backend: dream_extract_step file_catalog: dream topic_session_id: interests + scan_days: 2 - backend: dream_integrate_step - backend: dream_topics_step topic_count: 3 @@ -80,6 +81,10 @@ jobs: type: string description: "caller guidance passed through to dream extract/integrate" default: "" + scan_days: + type: integer + description: "number of recent daily directories to scan, ending at date" + default: 2 topic_count: type: integer description: "maximum number of final daily interest topics" @@ -92,6 +97,7 @@ jobs: - backend: dream_extract_step file_catalog: dream topic_session_id: interests + scan_days: 2 - backend: dream_integrate_step - backend: dream_topics_step topic_count: 3 diff --git a/reme/steps/evolve/dream/extract.py b/reme/steps/evolve/dream/extract.py index 0c9c6dcd..d82f1086 100644 --- a/reme/steps/evolve/dream/extract.py +++ b/reme/steps/evolve/dream/extract.py @@ -12,6 +12,7 @@ from .utils import ( llm_available, pack_paths, parse_structured_reply, + recent_dates, scan_day_files, store_state, today, @@ -25,34 +26,49 @@ _TOOLS = ("read",) class DreamExtractStep(BaseStep): """Scan changed daily files and globally extract merged units/topics.""" - def __init__(self, topic_session_id: str = "interests", **kwargs): + def __init__(self, topic_session_id: str = "interests", scan_days: int = 2, **kwargs): super().__init__(**kwargs) self.topic_session_id = topic_session_id + self.scan_days = scan_days async def execute(self): assert self.context is not None day = today(self, str(self.context.get("date", "") or "")) + raw_scan_days = self.context.get("scan_days", self.scan_days) + scan_days = max(int(raw_scan_days or self.scan_days), 1) + dates = recent_dates(day, scan_days) hint = str(self.context.get("hint", "") or "").strip() daily, workspace = daily_dir(self), workspace_dir(self) if self.file_catalog is None: raise RuntimeError("dream_extract_step requires file_catalog") - await refresh_day_index(self.file_store, day, daily) + for scan_day in dates: + await refresh_day_index(self.file_store, scan_day, daily) - existing = self._existing(workspace, scan_day_files(workspace, day, daily, f"{self.topic_session_id}.yaml")) - interests_rel = f"{daily}/{day}/{self.topic_session_id}.yaml" - day_md, day_prefix = f"{daily}/{day}.md", f"{daily}/{day}/" + existing = self._existing( + workspace, + [ + path + for scan_day in dates + for path in scan_day_files(workspace, scan_day, daily, f"{self.topic_session_id}.yaml") + ], + ) + interest_rels = {f"{daily}/{scan_day}/{self.topic_session_id}.yaml" for scan_day in dates} + day_mds = {f"{daily}/{scan_day}.md" for scan_day in dates} + day_prefixes = tuple(f"{daily}/{scan_day}/" for scan_day in dates) nodes = await self.file_catalog.get_nodes() - indexed_all = {n.path: n.st_mtime for n in nodes if n.path == day_md or n.path.startswith(day_prefix)} - indexed = {path: mt for path, mt in indexed_all.items() if path != interests_rel} + indexed_all = {n.path: n.st_mtime for n in nodes if n.path in day_mds or n.path.startswith(day_prefixes)} + indexed = {path: mt for path, mt in indexed_all.items() if path not in interest_rels} changed = [rel for rel, mt in existing.items() if indexed.get(rel) != mt] unchanged = [rel for rel, mt in existing.items() if indexed.get(rel) == mt] - protected = set(existing) | ({interests_rel} if (workspace / interests_rel).is_file() else set()) + protected = set(existing) | {rel for rel in interest_rels if (workspace / rel).is_file()} deleted = sorted(indexed_all.keys() - protected) if deleted: await self.file_catalog.delete(deleted) state = DreamState( date=day, + dates=dates, + scan_days=scan_days, hint=hint, daily_dir=daily, workspace=str(workspace), @@ -67,7 +83,7 @@ class DreamExtractStep(BaseStep): indexed=indexed, ) if not changed: - return self._finish(state, True, f"No changed dream input for {day}") + return self._finish(state, True, f"No changed dream input for {', '.join(dates)}") if not llm_available(self): state.errors.append("no llm configured; dream extract requires an LLM") return self._finish(state, False, state.errors[-1]) @@ -76,6 +92,7 @@ class DreamExtractStep(BaseStep): self.prompt_format( "extract_user_message", date=day, + dates_json=json.dumps(dates, ensure_ascii=False, indent=2), hint=hint or "(none)", changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), material_blob=pack_paths(workspace, changed), @@ -91,7 +108,7 @@ class DreamExtractStep(BaseStep): self._clean_output(state, meta) state.extract_summary = str(result.get("result") or "").strip() answer = f"Extracted {len(state.units)} unit(s), {len(state.topics)} topic(s)" - answer = f"{answer} from {len(changed)} changed file(s)" + answer = f"{answer} from {len(changed)} changed file(s) across {len(dates)} day(s)" return self._finish(state, True, answer) def _existing(self, workspace, files: list[str]) -> dict[str, float]: diff --git a/reme/steps/evolve/dream/extract.yaml b/reme/steps/evolve/dream/extract.yaml index 37da3fb6..487f7c07 100644 --- a/reme/steps/evolve/dream/extract.yaml +++ b/reme/steps/evolve/dream/extract.yaml @@ -152,6 +152,8 @@ extract_system_prompt_zh: | extract_user_message: | date: {date} + scan_dates: + {dates_json} hint: {hint} changed_paths: @@ -165,6 +167,8 @@ extract_user_message: | extract_user_message_zh: | 日期:{date} + 扫描日期: + {dates_json} 提示:{hint} changed_paths: diff --git a/reme/steps/evolve/dream/finish.py b/reme/steps/evolve/dream/finish.py index b790457a..97ba67c5 100644 --- a/reme/steps/evolve/dream/finish.py +++ b/reme/steps/evolve/dream/finish.py @@ -25,7 +25,9 @@ class DreamFinishStep(BaseStep): raise RuntimeError("dream_finish_step requires file_catalog") checkpoint = [p for p in state.changed_paths if p not in set(state.failed_paths)] - upsert_paths = checkpoint + [p for p in [state.interests_path, f"{state.daily_dir}/{state.date}.md"] if p] + day_index_paths = [f"{state.daily_dir}/{day}.md" for day in (state.dates or [state.date]) if day] + interest_paths = state.interests_paths or ([state.interests_path] if state.interests_path else []) + upsert_paths = checkpoint + [p for p in [*interest_paths, *day_index_paths] if p] upserts = self._nodes(workspace, upsert_paths) if upserts: await self.file_catalog.upsert(upserts) @@ -52,12 +54,14 @@ class DreamFinishStep(BaseStep): def render_summary(state: DreamState) -> str: """Render summary.""" + interest_paths = state.interests_paths or ([state.interests_path] if state.interests_path else []) lines = [ - f"[AutoDream] date={state.date} scanned={state.files_scanned} changed={state.files_changed} " + f"[AutoDream] date={state.date} dates={','.join(state.dates or [state.date])} " + f"scanned={state.files_scanned} changed={state.files_changed} " f"unchanged={state.files_unchanged} deleted={state.files_deleted}", f" - extract: {len(state.units)} unit(s), {len(state.topics)} topic(s)", f" - integrate: {len(state.integrate_results)} ok, {len(state.failed_units)} failed", - f" - topics: {state.topics_written} written" + (f" to {state.interests_path}" if state.interests_path else ""), + f" - topics: {state.topics_written} written" + (f" to {', '.join(interest_paths)}" if interest_paths else ""), f" - catalog: checkpointed {len(state.checkpoint_paths)} changed path(s)", ] if state.failed_paths: diff --git a/reme/steps/evolve/dream/schema.py b/reme/steps/evolve/dream/schema.py index 947ba123..e1d04f70 100644 --- a/reme/steps/evolve/dream/schema.py +++ b/reme/steps/evolve/dream/schema.py @@ -64,6 +64,8 @@ class DreamState(BaseModel): """Shared state passed across the four dream steps.""" date: str = "" + dates: list[str] = Field(default_factory=list) + scan_days: int = 2 hint: str = "" daily_dir: str = "" workspace: str = "" @@ -85,6 +87,7 @@ class DreamState(BaseModel): failed_units: list[dict] = Field(default_factory=list) failed_paths: list[str] = Field(default_factory=list) interests_path: str = "" + interests_paths: list[str] = Field(default_factory=list) topics_written: int = 0 topic_error: str = "" checkpoint_paths: list[str] = Field(default_factory=list) diff --git a/reme/steps/evolve/dream/topics.py b/reme/steps/evolve/dream/topics.py index f12af305..ff2d0df4 100644 --- a/reme/steps/evolve/dream/topics.py +++ b/reme/steps/evolve/dream/topics.py @@ -35,51 +35,87 @@ class DreamTopicsStep(BaseStep): raw_days = self.context.get("topic_diversity_days", self.topic_diversity_days) diversity_days = int(raw_days or self.topic_diversity_days) workspace = Path(state.workspace).resolve() if state.workspace else workspace_dir(self) - rel_path = f"{state.daily_dir}/{state.date}/interests.yaml" - abs_path = workspace / state.daily_dir / state.date / "interests.yaml" - same_day = load_yaml_topics(abs_path) + target_day = state.date or ((state.dates or [""])[-1]) if not state.topics: - state.interests_path = rel_path if abs_path.is_file() else "" - state.topics_written = len(same_day) - answer = f"Kept {len(same_day)} existing interest topic(s) at {rel_path}" - answer = answer if abs_path.is_file() else "Skipped interests.yaml write: no new topic candidates" + existing_paths = [] + if target_day and self._abs_path(workspace, state.daily_dir, target_day).is_file(): + existing_paths = [self._rel_path(state.daily_dir, target_day)] + state.interests_paths = existing_paths + state.interests_path = existing_paths[-1] if existing_paths else "" + state.topics_written = ( + len(load_yaml_topics(self._abs_path(workspace, state.daily_dir, target_day))) if target_day else 0 + ) + answer = ( + f"Kept existing interest topic(s) at {', '.join(existing_paths)}" + if existing_paths + else "Skipped interests.yaml write: no new topic candidates" + ) return self._finish(state, True, answer) - recent = [ - topic - for day in previous_dates(state.date, diversity_days) - for topic in load_yaml_topics(workspace / state.daily_dir / day / "interests.yaml") - ] try: - topics, _used_llm = await self._select_topics(state, same_day, recent, topic_count, diversity_days) + if not target_day: + state.interests_paths = [] + state.interests_path = "" + state.topics_written = 0 + return self._finish(state, True, "Skipped interests.yaml write: no target date") + + rel_path = self._rel_path(state.daily_dir, target_day) + abs_path = self._abs_path(workspace, state.daily_dir, target_day) + same_day = load_yaml_topics(abs_path) + recent = [ + topic + for previous_day in previous_dates(target_day, diversity_days) + for topic in load_yaml_topics(self._abs_path(workspace, state.daily_dir, previous_day)) + ] + topics, _used_llm = await self._select_topics( + state, + target_day, + state.topics, + same_day, + recent, + topic_count, + diversity_days, + ) payload = { - "date": state.date, + "date": target_day, "topic_count": topic_count, "diversity_days": diversity_days, "topics": topics, } write_yaml(abs_path, payload) - await refresh_day_index(self.file_store, state.date, state.daily_dir) - state.interests_path, state.topics_written = rel_path, len(topics) - return self._finish(state, True, f"Wrote {len(topics)} interest topic(s) to {rel_path}") + await refresh_day_index(self.file_store, target_day, state.daily_dir) + state.interests_paths = [rel_path] + state.interests_path = rel_path + state.topics_written = len(topics) + answer = f"Wrote {len(topics)} interest topic(s) to {rel_path}" + return self._finish(state, True, answer) except Exception as e: # noqa: BLE001 state.topic_error = f"{type(e).__name__}: {e}" state.errors.append(state.topic_error) return self._finish(state, False, f"Error: {state.topic_error}") - async def _select_topics(self, state, same_day: list[dict], recent: list[dict], count: int, days: int): - if not state.topics: + async def _select_topics( + self, + _state, + day: str, + candidates: list[dict], + same_day: list[dict], + recent: list[dict], + count: int, + days: int, + ): + if not candidates: return self._dedupe([], same_day, recent, count), False if not llm_available(self): - return self._dedupe(state.topics, same_day, recent, count), False + return self._dedupe(candidates, same_day, recent, count), False result = await self.agent_wrapper.reply( self.prompt_format( "topics_user_message", - date=state.date, + date=day, topic_count=count, diversity_days=days, - candidates_json=json.dumps(state.topics, ensure_ascii=False, indent=2), + candidates_json=json.dumps(candidates, ensure_ascii=False, indent=2), same_day_json=json.dumps(same_day, ensure_ascii=False, indent=2), recent_topics_json=json.dumps(recent, ensure_ascii=False, indent=2), ), @@ -88,9 +124,17 @@ class DreamTopicsStep(BaseStep): meta = parse_structured_reply(str(result.get("result") or "")) selected = [self._clean_topic(t) for t in meta.get("topics") or []] if not any(selected): - selected = state.topics + selected = candidates return self._dedupe(selected, same_day, recent, count), True + @staticmethod + def _rel_path(daily_dir: str, day: str) -> str: + return f"{daily_dir}/{day}/interests.yaml" + + @staticmethod + def _abs_path(workspace: Path, daily_dir: str, day: str) -> Path: + return workspace / daily_dir / day / "interests.yaml" + @staticmethod def _clean_topic(raw) -> dict: if not isinstance(raw, dict): diff --git a/reme/steps/evolve/dream/utils.py b/reme/steps/evolve/dream/utils.py index d1cc7ffa..428bf68d 100644 --- a/reme/steps/evolve/dream/utils.py +++ b/reme/steps/evolve/dream/utils.py @@ -48,6 +48,16 @@ def today(step: BaseStep, explicit: str = "") -> str: return now(tz).strftime("%Y-%m-%d") +def recent_dates(day: str, n_days: int) -> list[str]: + """Return the inclusive recent-date window ending at ``day``.""" + try: + base = dt.date.fromisoformat(day) + except ValueError: + return [day] if day else [] + n = max(int(n_days or 1), 1) + return [(base - dt.timedelta(days=i)).isoformat() for i in range(n - 1, -1, -1)] + + def llm_available(step: BaseStep) -> bool: """Check if LLM is available.""" try: diff --git a/tests/unit/test_auto_dream.py b/tests/unit/test_auto_dream.py index 06b62666..f5986266 100644 --- a/tests/unit/test_auto_dream.py +++ b/tests/unit/test_auto_dream.py @@ -4,11 +4,15 @@ import asyncio import tempfile from pathlib import Path +import yaml + from reme.components.file_catalog import BaseFileCatalog +from reme.components.file_store import BaseFileStore from reme.components.runtime_context import RuntimeContext from reme.steps.evolve.dream.finish import DreamFinishStep from reme.steps.evolve.dream.schema import DreamState -from reme.steps.evolve.dream.utils import parse_structured_reply, scan_day_files +from reme.steps.evolve.dream.topics import DreamTopicsStep +from reme.steps.evolve.dream.utils import parse_structured_reply, recent_dates, scan_day_files def _touch(path: Path, text: str = "x") -> Path: @@ -36,6 +40,40 @@ class _Catalog(BaseFileCatalog): self.dumps += 1 +class _FileStore(BaseFileStore): + def __init__(self, workspace: Path): + super().__init__() + self._workspace_path = workspace + + @property + def workspace_path(self) -> Path: + return self._workspace_path + + async def upsert(self, files): + return None + + async def delete(self, path): + return None + + async def clear(self): + return None + + async def get_nodes(self, paths=None): + return [] + + async def get_outlinks(self, path, scope=None): + return [] + + async def get_inlinks(self, path, scope=None): + return [] + + async def vector_search(self, query, limit, search_filter): + return [] + + async def keyword_search(self, query, limit, search_filter): + return [] + + def test_scan_day_files_includes_nested_md_and_excludes_interests(): """Scan day files.""" with tempfile.TemporaryDirectory() as tmp: @@ -52,6 +90,12 @@ def test_scan_day_files_includes_nested_md_and_excludes_interests(): ] +def test_recent_dates_includes_anchor_and_previous_days(): + """Recent date window is inclusive and chronological.""" + assert recent_dates("2026-05-28", 3) == ["2026-05-26", "2026-05-27", "2026-05-28"] + assert recent_dates("2026-05-28", 1) == ["2026-05-28"] + + def test_parse_structured_reply_handles_fenced_yaml_and_scalar_fallback(): """Parse a JSON/YAML object from an agent reply, including fenced blocks.""" data = parse_structured_reply( @@ -66,6 +110,48 @@ def test_parse_structured_reply_handles_fenced_yaml_and_scalar_fallback(): assert data["note"].startswith("Extended node") +def test_topics_step_writes_only_target_date_interests(): + """Topics are written only to ``state.date`` even when scan dates span multiple days.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) + _touch(workspace / "daily" / "2026-05-26" / "old.md") + _touch(workspace / "daily" / "2026-05-28" / "today.md") + old_interests = workspace / "daily" / "2026-05-26" / "interests.yaml" + _touch(old_interests, "date: 2026-05-26\ntopics: []\n") + state = DreamState( + date="2026-05-28", + dates=["2026-05-26", "2026-05-27", "2026-05-28"], + workspace=str(workspace), + daily_dir="daily", + topics=[ + { + "title": "Old changed topic", + "reason": "Old daily material changed.", + "paths": ["daily/2026-05-26/old.md"], + }, + { + "title": "Today changed topic", + "reason": "Today's daily material changed.", + "paths": ["daily/2026-05-28/today.md"], + }, + ], + ) + step = DreamTopicsStep() + resp = await step(RuntimeContext(dream=state.model_dump(), file_store=_FileStore(workspace))) + + target = workspace / "daily" / "2026-05-28" / "interests.yaml" + dream = resp.metadata["dream"] + assert resp.success is True + assert target.is_file() + assert old_interests.read_text(encoding="utf-8") == "date: 2026-05-26\ntopics: []\n" + assert dream["interests_paths"] == ["daily/2026-05-28/interests.yaml"] + assert yaml.safe_load(target.read_text(encoding="utf-8"))["date"] == "2026-05-28" + + asyncio.run(run()) + + def test_finish_does_not_checkpoint_failed_changed_paths(): """Finish does not checkpoint failed changed paths.""" @@ -74,13 +160,16 @@ def test_finish_does_not_checkpoint_failed_changed_paths(): workspace = Path(tmp) ok = _touch(workspace / "daily" / "2026-05-28" / "ok.md") failed = _touch(workspace / "daily" / "2026-05-28" / "failed.md") + day_index = _touch(workspace / "daily" / "2026-05-28.md") interests = _touch(workspace / "daily" / "2026-05-28" / "interests.yaml") state = DreamState( date="2026-05-28", + dates=["2026-05-26", "2026-05-27", "2026-05-28"], workspace=str(workspace), + daily_dir="daily", changed_paths=[str(ok.relative_to(workspace)), str(failed.relative_to(workspace))], failed_paths=[str(failed.relative_to(workspace))], - interests_path=str(interests.relative_to(workspace)), + interests_paths=[str(interests.relative_to(workspace))], ) step, catalog = DreamFinishStep(), _Catalog() resp = await step(RuntimeContext(dream=state.model_dump(), file_catalog=catalog)) @@ -90,6 +179,7 @@ def test_finish_does_not_checkpoint_failed_changed_paths(): assert str(ok.relative_to(workspace)) in upserted assert str(failed.relative_to(workspace)) not in upserted assert str(interests.relative_to(workspace)) in upserted + assert str(day_index.relative_to(workspace)) in upserted assert catalog.dumps == 1 asyncio.run(run()) diff --git a/tests/unit/test_file_store_consistency.py b/tests/unit/test_file_store_consistency.py index 12ed5517..6ddd1a97 100644 --- a/tests/unit/test_file_store_consistency.py +++ b/tests/unit/test_file_store_consistency.py @@ -90,6 +90,36 @@ def test_keyword_only_upsert_removes_old_chunks_and_docs(): run(go()) +def test_load_rebuilds_keyword_index_from_persisted_chunks_when_missing(): + """Loading persisted chunks repairs a missing keyword index.""" + + async def go(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(name="t_keyword_repair", embedding_store="") + await store.start() + + await store.upsert( + [ + (node("a.md"), [chunk("a", "a.md", "uniquerepairword stock")]), + (node("b.md"), [chunk("b", "b.md", "work preference")]), + ], + ) + await store.dump() + + await store.keyword_index.clear() + assert not store.keyword_index.index_file.exists() + assert await store.keyword_search("uniquerepairword", 5, {}) == [] + + store.file_chunks.clear() + await store.load() + + assert store.keyword_index.index_file.exists() + assert [c.id for c in await store.keyword_search("uniquerepairword", 5, {})] == ["a"] + await store.close() + + run(go()) + + def test_same_chunk_id_with_changed_text_gets_new_embedding(): """Changing a chunk text refreshes its embedding.""" diff --git a/tests/unit/test_keyword_index.py b/tests/unit/test_keyword_index.py index 2eb6923e..f8eba951 100644 --- a/tests/unit/test_keyword_index.py +++ b/tests/unit/test_keyword_index.py @@ -787,6 +787,47 @@ def test_index_file_isolated_by_tokenizer_config(): run(go()) +def test_tokenizer_fingerprint_ignores_stopwords_absolute_path(): + """Same stopwords content should not fork indexes by install path.""" + + async def go(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + stopwords_a = os.path.join(tmp, "a", "stopwords") + stopwords_b = os.path.join(tmp, "b", "stopwords") + os.makedirs(os.path.dirname(stopwords_a)) + os.makedirs(os.path.dirname(stopwords_b)) + with open(stopwords_a, "w", encoding="utf-8") as f: + f.write("alpha\nbeta\n") + with open(stopwords_b, "w", encoding="utf-8") as f: + f.write("alpha\nbeta\n") + + first = BM25Index() + first_tokenizer = RegexTokenizer(filter_stopwords=True, stopwords_path=stopwords_a) + first.tokenizer = first_tokenizer + first._owned.append(first_tokenizer) + await first.start() + + second = BM25Index() + second_tokenizer = RegexTokenizer(filter_stopwords=True, stopwords_path=stopwords_b) + second.tokenizer = second_tokenizer + second._owned.append(second_tokenizer) + await second.start() + + assert first._tokenizer_config()["stopwords_sha256"] == second._tokenizer_config()["stopwords_sha256"] + assert "stopwords_path" not in first._tokenizer_config() + assert first._tokenizer_fingerprint() == second._tokenizer_fingerprint() + assert first.index_file == second.index_file + + with open(stopwords_b, "w", encoding="utf-8") as f: + f.write("alpha\ngamma\n") + assert first._tokenizer_fingerprint() != second._tokenizer_fingerprint() + + await first.close() + await second.close() + + run(go()) + + def test_dump_failure_is_not_silent(): """A failed write must be observable by callers."""