mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
6a5561f86a
- Add file_graph import to component registry - Register FILE_GRAPH enum in ComponentEnum - Implement BaseFileGraph integration in LinkedFileParser - Replace FileEdge with FileLink for better semantic clarity - Add lazy resolution of file_graph from app_context - Update file watcher logging to reflect links instead of edges refactor: streamline MCP transport layer architecture - Remove redundant step shells from reme2/mcp/steps/ - Consolidate all @R.register components to reme2.memory package - Update server.py to import reme2.memory directly - Revise README.md to document new architecture - Simplify module dependencies and import structure
55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
"""Regex tokenizer implementation."""
|
|
|
|
import re
|
|
|
|
from .base_tokenizer import BaseTokenizer
|
|
from ..component_registry import R
|
|
|
|
|
|
@R.register("regex")
|
|
class RegexTokenizer(BaseTokenizer):
|
|
"""Tokenizer using regex for word segmentation, with Chinese character splitting."""
|
|
|
|
# Match words with word boundaries (2+ characters)
|
|
WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b")
|
|
# Match single Chinese character
|
|
CHINESE_PATTERN = re.compile(r"[一-鿿]")
|
|
|
|
def __init__(self, filter_stopwords: bool = True, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.filter_stopwords = filter_stopwords
|
|
|
|
def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
|
|
"""Tokenize texts using regex pattern.
|
|
|
|
Strategy:
|
|
1. Extract all Chinese characters (split by character)
|
|
2. Replace Chinese with spaces in original text
|
|
3. Extract non-Chinese words with word boundaries
|
|
|
|
Args:
|
|
texts: List of texts to tokenize.
|
|
lower: Whether to lowercase tokens.
|
|
|
|
Returns:
|
|
List of token lists. Note: tokens are unordered (Chinese chars first, then words).
|
|
"""
|
|
result = []
|
|
for text in texts:
|
|
tokens = []
|
|
|
|
# Extract all Chinese characters
|
|
tokens.extend(self.CHINESE_PATTERN.findall(text))
|
|
|
|
# Replace Chinese with spaces, then extract words
|
|
text_without_chinese = self.CHINESE_PATTERN.sub(" ", text)
|
|
tokens.extend(self.WORD_PATTERN.findall(text_without_chinese))
|
|
|
|
if lower:
|
|
tokens = [t.lower() for t in tokens]
|
|
|
|
if self.filter_stopwords and self._stopwords:
|
|
tokens = [t for t in tokens if t not in self._stopwords]
|
|
|
|
result.append(tokens)
|
|
return result
|