mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
25 lines
837 B
Python
25 lines
837 B
Python
"""Jieba tokenizer implementation."""
|
|
|
|
from .base_tokenizer import BaseTokenizer
|
|
from ..component_registry import R
|
|
|
|
|
|
@R.register("jieba")
|
|
class JiebaTokenizer(BaseTokenizer):
|
|
"""Tokenizer using jieba for Chinese text segmentation."""
|
|
|
|
def __init__(self, filter_stopwords: bool = True, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.filter_stopwords = filter_stopwords
|
|
|
|
def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
|
|
"""Tokenize texts using jieba."""
|
|
import jieba
|
|
|
|
result = []
|
|
for text in texts:
|
|
tokens = [x.lower() for x in jieba.cut(text)]
|
|
if self.filter_stopwords and self._stopwords:
|
|
tokens = [t for t in tokens if t not in self._stopwords]
|
|
result.append(tokens)
|
|
return result
|