diff --git a/reme2/component/file_store/bm25_lite.py b/reme2/component/file_store/bm25_lite.py index c81a206f..f35ceed4 100644 --- a/reme2/component/file_store/bm25_lite.py +++ b/reme2/component/file_store/bm25_lite.py @@ -109,13 +109,13 @@ class BM25Lite(BaseComponent): await self._tokenizer.start() if self.index_file.exists(): - await self.load() + self.load() self.logger.info(f"Loaded index from {self.index_dir}") async def _close(self) -> None: """Save index and cleanup tokenizer on shutdown.""" if self.inverted_index: - await self.dump() + self.dump() self.logger.info(f"Saved index to {self.index_dir}") if self._tokenizer is not None: @@ -240,7 +240,7 @@ class BM25Lite(BaseComponent): return dict(sorted(scores.items(), key=lambda x: x[1], reverse=True)[:k]) if scores else {} - async def dump(self): + def dump(self): """Persist index to disk via pickle.""" with open(self.index_file, "wb") as f: pickle.dump( @@ -255,7 +255,7 @@ class BM25Lite(BaseComponent): f, ) - async def load(self): + def load(self): """Load index from disk. Clears index on failure.""" try: with open(self.index_file, "rb") as f: diff --git a/reme2/component/tokenizer/jieba_tokenizer.py b/reme2/component/tokenizer/jieba_tokenizer.py index 47919a7f..a02f65d2 100644 --- a/reme2/component/tokenizer/jieba_tokenizer.py +++ b/reme2/component/tokenizer/jieba_tokenizer.py @@ -1,7 +1,5 @@ """Jieba tokenizer implementation.""" -import jieba - from .base_tokenizer import BaseTokenizer from ..component_registry import R @@ -16,10 +14,12 @@ class JiebaTokenizer(BaseTokenizer): def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]: """Tokenize texts using jieba.""" + import jieba + result = [] for text in texts: tokens = [x.lower() for x in jieba.cut(text)] if self.filter_stopwords and self._stopwords: tokens = [t for t in tokens if t not in self._stopwords] result.append(tokens) - return result \ No newline at end of file + return result diff --git a/test/reme2/test_bm25_lite.py b/test/reme2/test_bm25_lite.py index a0921b05..c242358b 100644 --- a/test/reme2/test_bm25_lite.py +++ b/test/reme2/test_bm25_lite.py @@ -2,12 +2,17 @@ import asyncio import tempfile +import warnings from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).parent.parent.parent)) +# Filter jieba/pkg_resources deprecation warnings +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + from reme2.component.file_store.bm25_lite import BM25Lite