up

reme2/component/file_store/base_file_store.py

@@ -120,7 +120,7 @@ class BaseFileStore(BaseComponent):
         """Delete the node entry for `path`. Chunks are managed separately."""
 
     @abstractmethod
-    async def read_node(self, path: str) -> FileNode | None:
+    async def get_node(self, path: str) -> FileNode | None:
         """Fetch a single node by path. None if absent."""
 
     @abstractmethod

reme2/component/file_store/local_file_store.py

@@ -87,7 +87,7 @@ class LocalFileStore(BaseFileStore):
     async def delete_node(self, path: str) -> None:
         self._nodes.pop(path, None)
 
-    async def read_node(self, path: str) -> FileNode | None:
+    async def get_node(self, path: str) -> FileNode | None:
         return self._nodes.get(path)
 
     # -- Chunk CRUD --------------------------------------------------------

reme2/component/tokenizer/__init__.py

@@ -0,0 +1,11 @@
+"""Tokenizer component module."""
+
+from .base_tokenizer import BaseTokenizer
+from .jieba_tokenizer import JiebaTokenizer
+from .regex_tokenizer import RegexTokenizer
+
+__all__ = [
+    "BaseTokenizer",
+    "JiebaTokenizer",
+    "RegexTokenizer",
+]

reme2/component/tokenizer/base_tokenizer.py

@@ -0,0 +1,49 @@
+"""Abstract base class for tokenizers."""
+
+import aiofiles
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+
+
+class BaseTokenizer(BaseComponent):
+    """Abstract base class for tokenizers.
+
+    Subclasses must implement the `tokenize` method.
+    The `_start` method loads stopwords, and `_close` clears them.
+    """
+
+    component_type = ComponentEnum.TOKENIZER
+
+    DEFAULT_STOPWORDS_PATH = Path(__file__).parent / "stopwords"
+
+    def __init__(self, stopwords_path: str | Path | None = None, **kwargs):
+        super().__init__(**kwargs)
+        self.stopwords_path = Path(stopwords_path) if stopwords_path else self.DEFAULT_STOPWORDS_PATH
+        self._stopwords: set[str] = set()
+
+    async def _start(self) -> None:
+        """Load stopwords from file."""
+        if self.stopwords_path.exists():
+            async with aiofiles.open(self.stopwords_path, encoding="utf-8") as f:
+                content = await f.read()
+            self._stopwords = set(line.strip().lower() for line in content.splitlines() if line.strip())
+            self.logger.info(f"Loaded {len(self._stopwords)} stopwords from {self.stopwords_path}")
+        else:
+            self.logger.warning(f"Stopwords file not found: {self.stopwords_path}")
+
+    async def _close(self) -> None:
+        """Clear stopwords."""
+        self._stopwords.clear()
+        self.logger.info("Cleared stopwords")
+
+    @property
+    def stopwords(self) -> set[str]:
+        """Get the loaded stopwords."""
+        return self._stopwords
+
+    @abstractmethod
+    def tokenize(self, texts: list[str], **kwargs) -> list[list[str]]:
+        """Tokenize a list of texts."""

reme2/component/tokenizer/jieba_tokenizer.py

@@ -0,0 +1,25 @@
+"""Jieba tokenizer implementation."""
+
+import jieba
+
+from .base_tokenizer import BaseTokenizer
+from ..component_registry import R
+
+
+@R.register("jieba")
+class JiebaTokenizer(BaseTokenizer):
+    """Tokenizer using jieba for Chinese text segmentation."""
+
+    def __init__(self, filter_stopwords: bool = True, **kwargs):
+        super().__init__(**kwargs)
+        self.filter_stopwords = filter_stopwords
+
+    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+        """Tokenize texts using jieba."""
+        result = []
+        for text in texts:
+            tokens = [x.lower() for x in jieba.cut(text)]
+            if self.filter_stopwords and self._stopwords:
+                tokens = [t for t in tokens if t not in self._stopwords]
+            result.append(tokens)
+        return result

reme2/component/tokenizer/regex_tokenizer.py

@@ -0,0 +1,55 @@
+"""Regex tokenizer implementation."""
+
+import re
+
+from .base_tokenizer import BaseTokenizer
+from ..component_registry import R
+
+
+@R.register("regex")
+class RegexTokenizer(BaseTokenizer):
+    """Tokenizer using regex for word segmentation, with Chinese character splitting."""
+
+    # Match words with word boundaries (2+ characters)
+    WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b")
+    # Match single Chinese character
+    CHINESE_PATTERN = re.compile(r"[一-鿿]")
+
+    def __init__(self, filter_stopwords: bool = True, **kwargs):
+        super().__init__(**kwargs)
+        self.filter_stopwords = filter_stopwords
+
+    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+        """Tokenize texts using regex pattern.
+
+        Strategy:
+        1. Extract all Chinese characters (split by character)
+        2. Replace Chinese with spaces in original text
+        3. Extract non-Chinese words with word boundaries
+
+        Args:
+            texts: List of texts to tokenize.
+            lower: Whether to lowercase tokens.
+
+        Returns:
+            List of token lists. Note: tokens are unordered (Chinese chars first, then words).
+        """
+        result = []
+        for text in texts:
+            tokens = []
+
+            # Extract all Chinese characters
+            tokens.extend(self.CHINESE_PATTERN.findall(text))
+
+            # Replace Chinese with spaces, then extract words
+            text_without_chinese = self.CHINESE_PATTERN.sub(" ", text)
+            tokens.extend(self.WORD_PATTERN.findall(text_without_chinese))
+
+            if lower:
+                tokens = [t.lower() for t in tokens]
+
+            if self.filter_stopwords and self._stopwords:
+                tokens = [t for t in tokens if t not in self._stopwords]
+
+            result.append(tokens)
+        return result

reme2/enumeration/component_enum.py

@@ -37,3 +37,5 @@ class ComponentEnum(str, Enum):
     STEP = "step"
 
     JOB = "job"
+
+    TOKENIZER = "tokenizer"

test/reme2/test_tokenizer.py

@@ -0,0 +1,174 @@
+"""Tests for Tokenizers."""
+
+import asyncio
+from pathlib import Path
+
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from reme2.component.tokenizer import JiebaTokenizer, RegexTokenizer
+
+
+async def compare_tokenizers(texts: list[str], filter_stopwords: bool = False, name: str = ""):
+    """Compare both tokenizers on same input."""
+    jieba = JiebaTokenizer(filter_stopwords=filter_stopwords)
+    regex = RegexTokenizer(filter_stopwords=filter_stopwords)
+
+    await jieba.start()
+    await regex.start()
+
+    jieba_result = jieba.tokenize(texts)
+    regex_result = regex.tokenize(texts)
+
+    print(f"\n--- {name} ---")
+    print(f"输入: {texts}")
+    print(f"Jieba: {jieba_result}")
+    print(f"Regex: {regex_result}")
+
+    await jieba.close()
+    await regex.close()
+
+    return jieba_result, regex_result
+
+
+def test_basic_chinese():
+    """Test basic Chinese text."""
+
+    async def run():
+        jieba_result, regex_result = await compare_tokenizers(
+            ["我爱北京天安门", "今天天气很好"],
+            name="纯中文",
+        )
+
+        assert "北京" in jieba_result[0] or "天安门" in jieba_result[0]
+        assert "我" in regex_result[0]
+        print("✓ test_basic_chinese passed")
+
+    asyncio.run(run())
+
+
+def test_basic_english():
+    """Test basic English text."""
+
+    async def run():
+        jieba_result, regex_result = await compare_tokenizers(
+            ["I love Beijing very much"],
+            name="英文",
+        )
+
+        assert "love" in regex_result[0]
+        assert "beijing" in regex_result[0]
+        print("✓ test_basic_english passed")
+
+    asyncio.run(run())
+
+
+def test_mixed_chinese_english():
+    """Test mixed Chinese-English text."""
+
+    async def run():
+        jieba_result, regex_result = await compare_tokenizers(
+            ["我用 Python 学习 machine learning 和 iPhone15 Pro。"],
+            name="中英混合",
+        )
+
+        assert "python" in jieba_result[0]
+        assert "python" in regex_result[0]
+        print("✓ test_mixed_chinese_english passed")
+
+    asyncio.run(run())
+
+
+def test_open_example():
+    """Test the 'open' example."""
+
+    async def run():
+        jieba_result, regex_result = await compare_tokenizers(
+            ["我觉得open很好呀，能分好次吗？"],
+            name="'open' 案例",
+        )
+
+        # open 保持完整
+        assert "open" in jieba_result[0]
+        assert "open" in regex_result[0]
+
+        # Regex 中文按字拆分
+        assert "我" in regex_result[0]
+        assert "很" in regex_result[0]
+
+        print("✓ test_open_example passed")
+
+    asyncio.run(run())
+
+
+def test_with_stopwords():
+    """Test with stopwords filtering."""
+
+    async def run():
+        jieba_result, regex_result = await compare_tokenizers(
+            ["我觉得open很好呀，能分好次吗？"],
+            filter_stopwords=True,
+            name="停用词过滤",
+        )
+
+        # 停用词被过滤
+        assert "吗" not in jieba_result[0]
+        assert "吗" not in regex_result[0]
+        assert "的" not in jieba_result[0]
+
+        print("✓ test_with_stopwords passed")
+
+    asyncio.run(run())
+
+
+def test_multiple_texts():
+    """Test multiple texts at once."""
+
+    async def run():
+        texts = [
+            "我爱北京天安门",
+            "I love Python programming",
+            "今天学习 machine learning",
+        ]
+        jieba_result, regex_result = await compare_tokenizers(texts, name="多个文本")
+
+        assert len(jieba_result) == 3
+        assert len(regex_result) == 3
+        print("✓ test_multiple_texts passed")
+
+    asyncio.run(run())
+
+
+def test_tokenizer_lifecycle():
+    """Test tokenizer start/close lifecycle."""
+
+    async def run():
+        tokenizer = JiebaTokenizer(filter_stopwords=True)
+
+        assert not tokenizer.is_started
+        assert len(tokenizer.stopwords) == 0
+
+        await tokenizer.start()
+        assert tokenizer.is_started
+        assert len(tokenizer.stopwords) > 0
+
+        await tokenizer.close()
+        assert not tokenizer.is_started
+        assert len(tokenizer.stopwords) == 0
+
+        print("✓ test_tokenizer_lifecycle passed")
+
+    asyncio.run(run())
+
+
+if __name__ == "__main__":
+    print("\n=== Tokenizer Tests ===")
+    test_basic_chinese()
+    test_basic_english()
+    test_mixed_chinese_english()
+    test_open_example()
+    test_with_stopwords()
+    test_multiple_texts()
+    test_tokenizer_lifecycle()
+    print("\n所有测试通过!")