diff --git a/reme4/components/file_store/base_file_store.py b/reme4/components/file_store/base_file_store.py index 7788487e..d7cc101e 100644 --- a/reme4/components/file_store/base_file_store.py +++ b/reme4/components/file_store/base_file_store.py @@ -18,13 +18,6 @@ class BaseFileStore(BaseComponent): component_type = ComponentEnum.FILE_STORE - def __init__(self, store_name: str, store_version: str = "v1", **kwargs): - super().__init__(**kwargs) - self.store_name = store_name or self.name - self.store_version = store_version - self.store_path = self.vault_metadata_path / self.component_type.value / self.store_name - self.store_path.mkdir(parents=True, exist_ok=True) - # -- CRUD ------------------------------------------------------------ @abstractmethod diff --git a/reme4/components/file_store/faiss_local_file_store.py b/reme4/components/file_store/faiss_local_file_store.py index d98ef718..af7f68b7 100644 --- a/reme4/components/file_store/faiss_local_file_store.py +++ b/reme4/components/file_store/faiss_local_file_store.py @@ -40,8 +40,8 @@ class FaissLocalFileStore(LocalFileStore): self._faiss = faiss self.normalize = normalize self.max_tombstones = max_tombstones - self.faiss_path = self.store_path / f"faiss_index_{self.store_version}.bin" - self.faiss_idmap_path = self.store_path / f"faiss_idmap_{self.store_version}.json" + self.faiss_path = self.component_metadata_path / f"faiss_index_{self.name}_{self.store_version}.bin" + self.faiss_idmap_path = self.component_metadata_path / f"faiss_idmap_{self.name}_{self.store_version}.json" self._faiss_index = None # faiss.Index | None self._id_map: list[str] = [] # row -> chunk_id self._id_to_row: dict[str, int] = {} # chunk_id -> row diff --git a/reme4/components/file_store/local_file_store.py b/reme4/components/file_store/local_file_store.py index 92c92e02..171aa66f 100644 --- a/reme4/components/file_store/local_file_store.py +++ b/reme4/components/file_store/local_file_store.py @@ -29,6 +29,7 @@ class LocalFileStore(BaseFileStore): keyword_index: str = "default", file_graph: str = "default", encoding: str = "utf-8", + store_version: str = "v1", **kwargs, ): super().__init__(**kwargs) @@ -46,15 +47,17 @@ class LocalFileStore(BaseFileStore): self.file_graph = self.bind(file_graph, BaseFileGraph, default_factory=LocalFileGraph) self.encoding = encoding + self.store_version = store_version + self.component_metadata_path.mkdir(parents=True, exist_ok=True) self.file_chunks: dict[str, FileChunk] = {} - self.chunks_path = self.store_path / f"file_chunks_{self.store_version}.jsonl" + self.chunks_path = self.component_metadata_path / f"file_chunks_{self.name}_{self.store_version}.jsonl" # Lifecycle async def _start(self) -> None: await super()._start() if self.embedding_model is not None and not await self.embedding_model.health_check(): - self.logger.warning(f"{self.store_name}: embedding unhealthy, vector disabled") + self.logger.warning(f"{self.name}: embedding unhealthy, vector disabled") self.embedding_model = None await self.load() @@ -67,7 +70,7 @@ class LocalFileStore(BaseFileStore): """Drop embedding after a runtime failure; keyword search still works.""" if self.embedding_model is None: return - self.logger.error(f"{self.store_name}: embedding disabled, {reason}") + self.logger.error(f"{self.name}: embedding disabled, {reason}") self.embedding_model = None async def load(self) -> None: diff --git a/reme4/components/tokenizer/base_tokenizer.py b/reme4/components/tokenizer/base_tokenizer.py index 713c2ecb..55a8c500 100644 --- a/reme4/components/tokenizer/base_tokenizer.py +++ b/reme4/components/tokenizer/base_tokenizer.py @@ -10,18 +10,28 @@ from ...enumeration import ComponentEnum class BaseTokenizer(BaseComponent): - """Base tokenizer. Subclasses must implement `tokenize`. Loads stopwords on start.""" + """Tokenizer base class with shared stopword loading and post-processing. + + Subclasses implement raw tokenization via `_tokenize_one`; lowercasing and + stopword filtering are handled here so every backend behaves consistently. + """ component_type = ComponentEnum.TOKENIZER DEFAULT_STOPWORDS_PATH = Path(__file__).parent / "stopwords" - def __init__(self, stopwords_path: str | Path | None = None, **kwargs): + def __init__( + self, + stopwords_path: str | Path | None = None, + filter_stopwords: bool = True, + **kwargs, + ): super().__init__(**kwargs) self.stopwords_path = Path(stopwords_path) if stopwords_path else self.DEFAULT_STOPWORDS_PATH + self.filter_stopwords = filter_stopwords self._stopwords: set[str] = set() async def _start(self) -> None: - """Load stopwords from file.""" + # A missing file is non-fatal: tokenizers still work, just without filtering. if not self.stopwords_path.exists(): self.logger.warning(f"Stopwords file not found: {self.stopwords_path}") return @@ -31,14 +41,23 @@ class BaseTokenizer(BaseComponent): self.logger.info(f"Loaded {len(self._stopwords)} stopwords from {self.stopwords_path}") async def _close(self) -> None: - """Clear stopwords.""" self._stopwords.clear() @property def stopwords(self) -> set[str]: - """Get the loaded stopwords.""" return self._stopwords + def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]: + """Tokenize each text and apply shared post-processing.""" + return [self._postprocess(self._tokenize_one(t, **kwargs), lower) for t in texts] + + def _postprocess(self, tokens: list[str], lower: bool) -> list[str]: + if lower: + tokens = [t.lower() for t in tokens] + if self.filter_stopwords and self._stopwords: + tokens = [t for t in tokens if t not in self._stopwords] + return tokens + @abstractmethod - def tokenize(self, texts: list[str], **kwargs) -> list[list[str]]: - """Tokenize a list of texts.""" + def _tokenize_one(self, text: str, **kwargs) -> list[str]: + """Return raw tokens for one text; lowercasing/filtering happen upstream.""" diff --git a/reme4/components/tokenizer/jieba_tokenizer.py b/reme4/components/tokenizer/jieba_tokenizer.py index 4391c89c..1dc98b13 100644 --- a/reme4/components/tokenizer/jieba_tokenizer.py +++ b/reme4/components/tokenizer/jieba_tokenizer.py @@ -6,22 +6,10 @@ from ..component_registry import R @R.register("jieba") class JiebaTokenizer(BaseTokenizer): - """Tokenizer using jieba for Chinese text segmentation.""" + """Tokenizer backed by jieba for Chinese word segmentation.""" - def __init__(self, filter_stopwords: bool = True, **kwargs): - super().__init__(**kwargs) - self.filter_stopwords = filter_stopwords - - def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]: - """Tokenize texts using jieba.""" + def _tokenize_one(self, text: str, **kwargs) -> list[str]: + # Lazy import: jieba startup cost is non-trivial and only paid when used. import jieba - result = [] - for text in texts: - tokens = jieba.cut(text) - if lower: - tokens = [x.lower() for x in tokens] - if self.filter_stopwords and self._stopwords: - tokens = [t for t in tokens if t not in self._stopwords] - result.append(tokens) - return result + return list(jieba.cut(text)) diff --git a/reme4/components/tokenizer/regex_tokenizer.py b/reme4/components/tokenizer/regex_tokenizer.py index 4379a60e..a5b8be11 100644 --- a/reme4/components/tokenizer/regex_tokenizer.py +++ b/reme4/components/tokenizer/regex_tokenizer.py @@ -1,31 +1,24 @@ """Regex tokenizer with Chinese character splitting.""" import re + from .base_tokenizer import BaseTokenizer from ..component_registry import R @R.register("regex") class RegexTokenizer(BaseTokenizer): - """Tokenizer using regex: splits Chinese chars individually, extracts non-Chinese words.""" + """Regex tokenizer: each CJK char is its own token, non-CJK uses word boundaries. - WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b") # 2+ char words - CHINESE_PATTERN = re.compile(r"[一-鿿]") # single Chinese char + Treating CJK characters as individual tokens avoids needing a Chinese + segmenter while still giving BM25-style indexes useful unigrams. + """ - def __init__(self, filter_stopwords: bool = True, **kwargs): - super().__init__(**kwargs) - self.filter_stopwords = filter_stopwords + WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b") # non-CJK words, 2+ chars + CHINESE_PATTERN = re.compile(r"[一-鿿]") - def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]: - """Tokenize texts. Extracts Chinese chars, then non-Chinese words from remaining text.""" - result = [] - for text in texts: - # Extract Chinese chars individually, then non-Chinese words - tokens = self.CHINESE_PATTERN.findall(text) - tokens.extend(self.WORD_PATTERN.findall(self.CHINESE_PATTERN.sub(" ", text))) - if lower: - tokens = [t.lower() for t in tokens] - if self.filter_stopwords and self._stopwords: - tokens = [t for t in tokens if t not in self._stopwords] - result.append(tokens) - return result + def _tokenize_one(self, text: str, **kwargs) -> list[str]: + # Pull CJK chars first, then strip them out so the word regex only sees the rest. + tokens = self.CHINESE_PATTERN.findall(text) + tokens.extend(self.WORD_PATTERN.findall(self.CHINESE_PATTERN.sub(" ", text))) + return tokens