up

reme2/component/file_store/bm25_lite.py

@@ -109,13 +109,13 @@ class BM25Lite(BaseComponent):
             await self._tokenizer.start()
 
         if self.index_file.exists():
-            await self.load()
+            self.load()
             self.logger.info(f"Loaded index from {self.index_dir}")
 
     async def _close(self) -> None:
         """Save index and cleanup tokenizer on shutdown."""
         if self.inverted_index:
-            await self.dump()
+            self.dump()
             self.logger.info(f"Saved index to {self.index_dir}")
 
         if self._tokenizer is not None:
@@ -240,7 +240,7 @@ class BM25Lite(BaseComponent):
 
         return dict(sorted(scores.items(), key=lambda x: x[1], reverse=True)[:k]) if scores else {}
 
-    async def dump(self):
+    def dump(self):
         """Persist index to disk via pickle."""
         with open(self.index_file, "wb") as f:
             pickle.dump(
@@ -255,7 +255,7 @@ class BM25Lite(BaseComponent):
                 f,
             )
 
-    async def load(self):
+    def load(self):
         """Load index from disk. Clears index on failure."""
         try:
             with open(self.index_file, "rb") as f:

reme2/component/tokenizer/jieba_tokenizer.py

@@ -1,7 +1,5 @@
 """Jieba tokenizer implementation."""
 
-import jieba
-
 from .base_tokenizer import BaseTokenizer
 from ..component_registry import R
 
@@ -16,10 +14,12 @@ class JiebaTokenizer(BaseTokenizer):
 
     def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
         """Tokenize texts using jieba."""
+        import jieba
+
         result = []
         for text in texts:
             tokens = [x.lower() for x in jieba.cut(text)]
             if self.filter_stopwords and self._stopwords:
                 tokens = [t for t in tokens if t not in self._stopwords]
             result.append(tokens)
-        return result
+        return result

test/reme2/test_bm25_lite.py

@@ -2,12 +2,17 @@
 
 import asyncio
 import tempfile
+import warnings
 from pathlib import Path
 
 import sys
 
 sys.path.insert(0, str(Path(__file__).parent.parent.parent))
 
+# Filter jieba/pkg_resources deprecation warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
+
 from reme2.component.file_store.bm25_lite import BM25Lite