up

reme4/components/file_store/base_file_store.py

@@ -18,13 +18,6 @@ class BaseFileStore(BaseComponent):
 
     component_type = ComponentEnum.FILE_STORE
 
-    def __init__(self, store_name: str, store_version: str = "v1", **kwargs):
-        super().__init__(**kwargs)
-        self.store_name = store_name or self.name
-        self.store_version = store_version
-        self.store_path = self.vault_metadata_path / self.component_type.value / self.store_name
-        self.store_path.mkdir(parents=True, exist_ok=True)
-
     # -- CRUD ------------------------------------------------------------
 
     @abstractmethod

reme4/components/file_store/faiss_local_file_store.py

@@ -40,8 +40,8 @@ class FaissLocalFileStore(LocalFileStore):
         self._faiss = faiss
         self.normalize = normalize
         self.max_tombstones = max_tombstones
-        self.faiss_path = self.store_path / f"faiss_index_{self.store_version}.bin"
-        self.faiss_idmap_path = self.store_path / f"faiss_idmap_{self.store_version}.json"
+        self.faiss_path = self.component_metadata_path / f"faiss_index_{self.name}_{self.store_version}.bin"
+        self.faiss_idmap_path = self.component_metadata_path / f"faiss_idmap_{self.name}_{self.store_version}.json"
         self._faiss_index = None  # faiss.Index | None
         self._id_map: list[str] = []  # row -> chunk_id
         self._id_to_row: dict[str, int] = {}  # chunk_id -> row

reme4/components/file_store/local_file_store.py

@@ -29,6 +29,7 @@ class LocalFileStore(BaseFileStore):
         keyword_index: str = "default",
         file_graph: str = "default",
         encoding: str = "utf-8",
+        store_version: str = "v1",
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -46,15 +47,17 @@ class LocalFileStore(BaseFileStore):
         self.file_graph = self.bind(file_graph, BaseFileGraph, default_factory=LocalFileGraph)
 
         self.encoding = encoding
+        self.store_version = store_version
+        self.component_metadata_path.mkdir(parents=True, exist_ok=True)
         self.file_chunks: dict[str, FileChunk] = {}
-        self.chunks_path = self.store_path / f"file_chunks_{self.store_version}.jsonl"
+        self.chunks_path = self.component_metadata_path / f"file_chunks_{self.name}_{self.store_version}.jsonl"
 
     # Lifecycle
 
     async def _start(self) -> None:
         await super()._start()
         if self.embedding_model is not None and not await self.embedding_model.health_check():
-            self.logger.warning(f"{self.store_name}: embedding unhealthy, vector disabled")
+            self.logger.warning(f"{self.name}: embedding unhealthy, vector disabled")
             self.embedding_model = None
         await self.load()
 
@@ -67,7 +70,7 @@ class LocalFileStore(BaseFileStore):
         """Drop embedding after a runtime failure; keyword search still works."""
         if self.embedding_model is None:
             return
-        self.logger.error(f"{self.store_name}: embedding disabled, {reason}")
+        self.logger.error(f"{self.name}: embedding disabled, {reason}")
         self.embedding_model = None
 
     async def load(self) -> None:

reme4/components/tokenizer/base_tokenizer.py

@@ -10,18 +10,28 @@ from ...enumeration import ComponentEnum
 
 
 class BaseTokenizer(BaseComponent):
-    """Base tokenizer. Subclasses must implement `tokenize`. Loads stopwords on start."""
+    """Tokenizer base class with shared stopword loading and post-processing.
+
+    Subclasses implement raw tokenization via `_tokenize_one`; lowercasing and
+    stopword filtering are handled here so every backend behaves consistently.
+    """
 
     component_type = ComponentEnum.TOKENIZER
     DEFAULT_STOPWORDS_PATH = Path(__file__).parent / "stopwords"
 
-    def __init__(self, stopwords_path: str | Path | None = None, **kwargs):
+    def __init__(
+        self,
+        stopwords_path: str | Path | None = None,
+        filter_stopwords: bool = True,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self.stopwords_path = Path(stopwords_path) if stopwords_path else self.DEFAULT_STOPWORDS_PATH
+        self.filter_stopwords = filter_stopwords
         self._stopwords: set[str] = set()
 
     async def _start(self) -> None:
-        """Load stopwords from file."""
+        # A missing file is non-fatal: tokenizers still work, just without filtering.
         if not self.stopwords_path.exists():
             self.logger.warning(f"Stopwords file not found: {self.stopwords_path}")
             return
@@ -31,14 +41,23 @@ class BaseTokenizer(BaseComponent):
         self.logger.info(f"Loaded {len(self._stopwords)} stopwords from {self.stopwords_path}")
 
     async def _close(self) -> None:
-        """Clear stopwords."""
         self._stopwords.clear()
 
     @property
     def stopwords(self) -> set[str]:
-        """Get the loaded stopwords."""
         return self._stopwords
 
+    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+        """Tokenize each text and apply shared post-processing."""
+        return [self._postprocess(self._tokenize_one(t, **kwargs), lower) for t in texts]
+
+    def _postprocess(self, tokens: list[str], lower: bool) -> list[str]:
+        if lower:
+            tokens = [t.lower() for t in tokens]
+        if self.filter_stopwords and self._stopwords:
+            tokens = [t for t in tokens if t not in self._stopwords]
+        return tokens
+
     @abstractmethod
-    def tokenize(self, texts: list[str], **kwargs) -> list[list[str]]:
-        """Tokenize a list of texts."""
+    def _tokenize_one(self, text: str, **kwargs) -> list[str]:
+        """Return raw tokens for one text; lowercasing/filtering happen upstream."""

reme4/components/tokenizer/jieba_tokenizer.py

@@ -6,22 +6,10 @@ from ..component_registry import R
 
 @R.register("jieba")
 class JiebaTokenizer(BaseTokenizer):
-    """Tokenizer using jieba for Chinese text segmentation."""
+    """Tokenizer backed by jieba for Chinese word segmentation."""
 
-    def __init__(self, filter_stopwords: bool = True, **kwargs):
-        super().__init__(**kwargs)
-        self.filter_stopwords = filter_stopwords
-
-    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
-        """Tokenize texts using jieba."""
+    def _tokenize_one(self, text: str, **kwargs) -> list[str]:
+        # Lazy import: jieba startup cost is non-trivial and only paid when used.
         import jieba
 
-        result = []
-        for text in texts:
-            tokens = jieba.cut(text)
-            if lower:
-                tokens = [x.lower() for x in tokens]
-            if self.filter_stopwords and self._stopwords:
-                tokens = [t for t in tokens if t not in self._stopwords]
-            result.append(tokens)
-        return result
+        return list(jieba.cut(text))

reme4/components/tokenizer/regex_tokenizer.py

@@ -1,31 +1,24 @@
 """Regex tokenizer with Chinese character splitting."""
 
 import re
+
 from .base_tokenizer import BaseTokenizer
 from ..component_registry import R
 
 
 @R.register("regex")
 class RegexTokenizer(BaseTokenizer):
-    """Tokenizer using regex: splits Chinese chars individually, extracts non-Chinese words."""
+    """Regex tokenizer: each CJK char is its own token, non-CJK uses word boundaries.
 
-    WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b")  # 2+ char words
-    CHINESE_PATTERN = re.compile(r"[一-鿿]")  # single Chinese char
+    Treating CJK characters as individual tokens avoids needing a Chinese
+    segmenter while still giving BM25-style indexes useful unigrams.
+    """
 
-    def __init__(self, filter_stopwords: bool = True, **kwargs):
-        super().__init__(**kwargs)
-        self.filter_stopwords = filter_stopwords
+    WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b")  # non-CJK words, 2+ chars
+    CHINESE_PATTERN = re.compile(r"[一-鿿]")
 
-    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
-        """Tokenize texts. Extracts Chinese chars, then non-Chinese words from remaining text."""
-        result = []
-        for text in texts:
-            # Extract Chinese chars individually, then non-Chinese words
-            tokens = self.CHINESE_PATTERN.findall(text)
-            tokens.extend(self.WORD_PATTERN.findall(self.CHINESE_PATTERN.sub(" ", text)))
-            if lower:
-                tokens = [t.lower() for t in tokens]
-            if self.filter_stopwords and self._stopwords:
-                tokens = [t for t in tokens if t not in self._stopwords]
-            result.append(tokens)
-        return result
+    def _tokenize_one(self, text: str, **kwargs) -> list[str]:
+        # Pull CJK chars first, then strip them out so the word regex only sees the rest.
+        tokens = self.CHINESE_PATTERN.findall(text)
+        tokens.extend(self.WORD_PATTERN.findall(self.CHINESE_PATTERN.sub(" ", text)))
+        return tokens