From 29688d684556c294a621ac5e51c69985cdc04ef8 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Fri, 15 May 2026 23:31:01 +0800 Subject: [PATCH] up --- reme4/application.py | 4 +++ .../reme_openai_chat_formatter.py | 10 +++--- .../estimate_token_counter.py | 2 +- reme4/components/base_component.py | 8 +++-- .../components/file_graph/base_file_graph.py | 2 ++ reme4/components/file_parser/__init__.py | 2 ++ .../file_parser/bare_file_parser.py | 2 ++ .../file_parser/base_file_parser.py | 2 ++ .../file_parser/default_file_parser.py | 4 ++- .../components/file_store/base_file_store.py | 7 ++++ .../file_watcher/base_file_watcher.py | 2 ++ .../file_watcher/lite_file_watcher.py | 2 ++ reme4/components/keyword_index/__init__.py | 2 ++ reme4/components/runtime_context.py | 8 +++++ reme4/components/service/base_service.py | 13 ++++++-- reme4/components/service/http_service.py | 2 ++ reme4/components/service/mcp_service.py | 2 ++ reme4/components/tokenizer/base_tokenizer.py | 3 +- reme4/components/tokenizer/jieba_tokenizer.py | 4 ++- reme4/reme.py | 3 ++ reme4/schema/emb_node.py | 2 ++ reme4/schema/file_link.py | 2 ++ reme4/steps/base_step.py | 32 ++++++++++++------- 23 files changed, 96 insertions(+), 24 deletions(-) diff --git a/reme4/application.py b/reme4/application.py index f98eaa02..913f1c4c 100644 --- a/reme4/application.py +++ b/reme4/application.py @@ -1,3 +1,5 @@ +"""Main application entry point.""" + import asyncio import heapq from pathlib import Path @@ -69,6 +71,7 @@ class Application(BaseComponent): @property def config(self): + """Application configuration.""" return self.context.app_config def _topological_order(self) -> list[BaseComponent]: @@ -162,4 +165,5 @@ class Application(BaseComponent): yield chunk def run_app(self): + """Start the service and serve the application.""" self.context.service.run_app(app=self) diff --git a/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py b/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py index a0b47082..a40f977d 100644 --- a/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py +++ b/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py @@ -4,11 +4,13 @@ import json from typing import Any from agentscope.formatter import OpenAIChatFormatter -from agentscope.formatter import _openai_formatter as _of -_format_openai_image_block = getattr(_of, "_format_openai_image_block") -_to_openai_audio_data = getattr(_of, "_to_openai_audio_data") -from agentscope.message import ImageBlock, Msg, TextBlock, URLSource +# noinspection PyProtectedMember +from agentscope.formatter._openai_formatter import ( + _format_openai_image_block, + _to_openai_audio_data, +) +from agentscope.message import Msg, TextBlock, ImageBlock, URLSource def _format_openai_video_block(video_block: dict) -> dict[str, Any]: diff --git a/reme4/components/as_token_counter/estimate_token_counter.py b/reme4/components/as_token_counter/estimate_token_counter.py index 7a7e4fbe..b5566dec 100644 --- a/reme4/components/as_token_counter/estimate_token_counter.py +++ b/reme4/components/as_token_counter/estimate_token_counter.py @@ -16,6 +16,6 @@ class EstimatedTokenCounter(TokenCounterBase): self.estimate_divisor: float = estimate_divisor self.encoding: str = encoding - async def count(self, text: str, **kwargs) -> int: + async def count(self, text: str, **_kwargs) -> int: """Estimated token count for ``text``.""" return int(len(text.encode(self.encoding)) / self.estimate_divisor + 0.5) diff --git a/reme4/components/base_component.py b/reme4/components/base_component.py index 97157ae5..afbcb86f 100644 --- a/reme4/components/base_component.py +++ b/reme4/components/base_component.py @@ -3,12 +3,14 @@ import asyncio from abc import ABC from pathlib import Path -from typing import Any, Callable, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast -from .application_context import ApplicationContext from ..enumeration import ComponentEnum from ..utils import get_logger +if TYPE_CHECKING: + from .application_context import ApplicationContext + T = TypeVar("T", bound="BaseComponent") @@ -67,6 +69,7 @@ class BaseComponent(ABC): @property def is_started(self) -> bool: + """Whether the component has been started.""" return self._is_started # ----- Dependency declaration ---------------------------------------- @@ -119,6 +122,7 @@ class BaseComponent(ABC): @property def working_path(self) -> Path: + """Resolved working directory from app context or cwd.""" if self.app_context is None: return Path.cwd() return Path(self.app_context.app_config.working_dir) diff --git a/reme4/components/file_graph/base_file_graph.py b/reme4/components/file_graph/base_file_graph.py index 7ae47ec7..6d04404f 100644 --- a/reme4/components/file_graph/base_file_graph.py +++ b/reme4/components/file_graph/base_file_graph.py @@ -1,3 +1,5 @@ +"""Abstract base for file-graph backends.""" + from abc import abstractmethod from pathlib import Path diff --git a/reme4/components/file_parser/__init__.py b/reme4/components/file_parser/__init__.py index 8898bec4..3e20d0bf 100644 --- a/reme4/components/file_parser/__init__.py +++ b/reme4/components/file_parser/__init__.py @@ -1,3 +1,5 @@ +"""File parser components.""" + from .bare_file_parser import BareFileParser from .base_file_parser import BaseFileParser from .default_file_parser import DefaultFileParser diff --git a/reme4/components/file_parser/bare_file_parser.py b/reme4/components/file_parser/bare_file_parser.py index daadccb9..e0ecef05 100644 --- a/reme4/components/file_parser/bare_file_parser.py +++ b/reme4/components/file_parser/bare_file_parser.py @@ -1,3 +1,5 @@ +"""Stat-only parser for attachment/binary files.""" + from pathlib import Path from .base_file_parser import BaseFileParser diff --git a/reme4/components/file_parser/base_file_parser.py b/reme4/components/file_parser/base_file_parser.py index 2433a831..30595385 100644 --- a/reme4/components/file_parser/base_file_parser.py +++ b/reme4/components/file_parser/base_file_parser.py @@ -1,3 +1,5 @@ +"""Abstract base for file parsers.""" + from abc import abstractmethod from pathlib import Path diff --git a/reme4/components/file_parser/default_file_parser.py b/reme4/components/file_parser/default_file_parser.py index 53bb10ec..d64b0234 100644 --- a/reme4/components/file_parser/default_file_parser.py +++ b/reme4/components/file_parser/default_file_parser.py @@ -1,3 +1,5 @@ +"""Default file parser with byte-based overlapping chunking.""" + from bisect import bisect_right from pathlib import Path @@ -69,7 +71,7 @@ class DefaultFileParser(BaseFileParser): if content_bytes[end - 1] == ord("\n"): end_line -= 1 chunks.append( - FileChunk(path=rel_path, start_line=start_line, end_line=end_line, text=chunk_text).set_hash_id() + FileChunk(path=rel_path, start_line=start_line, end_line=end_line, text=chunk_text).set_hash_id(), ) if end >= len(content_bytes): break diff --git a/reme4/components/file_store/base_file_store.py b/reme4/components/file_store/base_file_store.py index 05a132fd..4f407145 100644 --- a/reme4/components/file_store/base_file_store.py +++ b/reme4/components/file_store/base_file_store.py @@ -1,3 +1,5 @@ +"""Abstract base for file store backends.""" + from abc import abstractmethod from ..base_component import BaseComponent @@ -9,6 +11,8 @@ from ...schema import FileChunk, FileNode, FileLink class BaseFileStore(BaseComponent): + """Abstract base for file store backends.""" + component_type = ComponentEnum.FILE_STORE def __init__( @@ -51,16 +55,19 @@ class BaseFileStore(BaseComponent): """Perform full-text keyword search.""" async def rebuild_links(self) -> None: + """Rebuild all edges from each node's link payload.""" if not self.file_graph: raise RuntimeError("file_graph is required for delete_by_path") return await self.file_graph.rebuild_links() async def get_outlinks(self, path: str) -> list[FileLink]: + """Return outgoing links for *path*.""" if not self.file_graph: raise RuntimeError("file_graph is required for delete_by_path") return await self.file_graph.get_outlinks(path) async def get_inlinks(self, path: str) -> list[FileLink]: + """Return incoming links for *path*.""" if not self.file_graph: raise RuntimeError("file_graph is required for delete_by_path") return await self.file_graph.get_inlinks(path) diff --git a/reme4/components/file_watcher/base_file_watcher.py b/reme4/components/file_watcher/base_file_watcher.py index 6424ef5e..79e111dd 100644 --- a/reme4/components/file_watcher/base_file_watcher.py +++ b/reme4/components/file_watcher/base_file_watcher.py @@ -1,3 +1,5 @@ +"""Abstract base for file watchers.""" + import asyncio from abc import abstractmethod from pathlib import Path diff --git a/reme4/components/file_watcher/lite_file_watcher.py b/reme4/components/file_watcher/lite_file_watcher.py index 1d50711e..633a1e82 100644 --- a/reme4/components/file_watcher/lite_file_watcher.py +++ b/reme4/components/file_watcher/lite_file_watcher.py @@ -1,3 +1,5 @@ +"""Polling-based file watcher using watchfiles.""" + import asyncio from pathlib import Path diff --git a/reme4/components/keyword_index/__init__.py b/reme4/components/keyword_index/__init__.py index fc08222c..f9969c6a 100644 --- a/reme4/components/keyword_index/__init__.py +++ b/reme4/components/keyword_index/__init__.py @@ -1,3 +1,5 @@ +"""Keyword index components.""" + from .base_keyword_index import BaseKeywordIndex from .bm25_index import BM25Index diff --git a/reme4/components/runtime_context.py b/reme4/components/runtime_context.py index 92ee2b79..76ebaf16 100644 --- a/reme4/components/runtime_context.py +++ b/reme4/components/runtime_context.py @@ -24,9 +24,11 @@ class RuntimeContext: self.data: dict = kwargs def get(self, key: str, default=None): + """Get a value from the data dict.""" return self.data.get(key, default) def update(self, data: dict) -> "RuntimeContext": + """Merge data into the context.""" self.data.update(data) return self @@ -44,10 +46,12 @@ class RuntimeContext: @property def stream(self) -> bool: + """Whether streaming is enabled.""" return self.stream_queue is not None @classmethod def from_context(cls, context: "RuntimeContext | None" = None, **kwargs) -> "RuntimeContext": + """Reuse or create a RuntimeContext.""" # Reuse the existing context (merging kwargs) or create a new one. if context is None: return cls(**kwargs) @@ -55,21 +59,25 @@ class RuntimeContext: return context async def _enqueue(self, chunk: StreamChunk) -> None: + """Put a chunk on the stream queue.""" if self.stream_queue is None: raise RuntimeError("Stream queue not initialized") await self.stream_queue.put(chunk) async def add_stream_string(self, chunk: str, chunk_type: ChunkEnum) -> "RuntimeContext": + """Emit a text chunk to the stream queue.""" # Emit a text chunk to the stream queue. await self._enqueue(StreamChunk(chunk_type=chunk_type, chunk=chunk)) return self async def add_stream_done(self) -> "RuntimeContext": + """Emit the terminal DONE marker to close the stream.""" # Emit the terminal DONE marker to close the stream. await self._enqueue(StreamChunk(chunk_type=ChunkEnum.DONE, chunk="", done=True)) return self def apply_mapping(self, mapping: dict[str, str]) -> "RuntimeContext": + """Copy data[source] into data[target] for each mapping pair.""" # Copy data[source] into data[target] for each {source: target} pair. if not mapping: return self diff --git a/reme4/components/service/base_service.py b/reme4/components/service/base_service.py index e47e25dd..8fceeadb 100644 --- a/reme4/components/service/base_service.py +++ b/reme4/components/service/base_service.py @@ -1,3 +1,5 @@ +"""Base service class for exposing jobs via HTTP, MCP, etc.""" + from abc import abstractmethod from typing import TYPE_CHECKING @@ -19,15 +21,19 @@ class BaseService(BaseComponent): self.service = None @abstractmethod - def build_service(self, app: "Application") -> None: ... + def build_service(self, app: "Application") -> None: + """Initialize the underlying service framework.""" @abstractmethod - def add_job(self, job: BaseJob) -> None: ... + def add_job(self, job: BaseJob) -> None: + """Register a single job with the service.""" @abstractmethod - def start_service(self, app: "Application") -> None: ... + def start_service(self, app: "Application") -> None: + """Start serving requests.""" def add_jobs(self, app: "Application") -> None: + """Register all jobs from the application context.""" for name, job in app.context.jobs.items(): try: self.add_job(job) @@ -36,6 +42,7 @@ class BaseService(BaseComponent): self.logger.error(f"Failed to add job {name}: {e}") def run_app(self, app: "Application") -> None: + """Build, populate, and start the service.""" self.build_service(app) self.add_jobs(app) self.start_service(app) diff --git a/reme4/components/service/http_service.py b/reme4/components/service/http_service.py index 83ed7105..6c7b9272 100644 --- a/reme4/components/service/http_service.py +++ b/reme4/components/service/http_service.py @@ -1,3 +1,5 @@ +"""HTTP service implementation for ReMe.""" + import asyncio import json import os diff --git a/reme4/components/service/mcp_service.py b/reme4/components/service/mcp_service.py index f14dfa96..8f22450d 100644 --- a/reme4/components/service/mcp_service.py +++ b/reme4/components/service/mcp_service.py @@ -1,3 +1,5 @@ +"""MCP (Model Context Protocol) service implementation.""" + import json import os from contextlib import asynccontextmanager diff --git a/reme4/components/tokenizer/base_tokenizer.py b/reme4/components/tokenizer/base_tokenizer.py index ee652cb3..713c2ecb 100644 --- a/reme4/components/tokenizer/base_tokenizer.py +++ b/reme4/components/tokenizer/base_tokenizer.py @@ -1,9 +1,10 @@ """Abstract base class for tokenizers.""" -import aiofiles from abc import abstractmethod from pathlib import Path +import aiofiles + from ..base_component import BaseComponent from ...enumeration import ComponentEnum diff --git a/reme4/components/tokenizer/jieba_tokenizer.py b/reme4/components/tokenizer/jieba_tokenizer.py index b4f9219f..4391c89c 100644 --- a/reme4/components/tokenizer/jieba_tokenizer.py +++ b/reme4/components/tokenizer/jieba_tokenizer.py @@ -18,7 +18,9 @@ class JiebaTokenizer(BaseTokenizer): result = [] for text in texts: - tokens = [x.lower() for x in jieba.cut(text)] + tokens = jieba.cut(text) + if lower: + tokens = [x.lower() for x in tokens] if self.filter_stopwords and self._stopwords: tokens = [t for t in tokens if t not in self._stopwords] result.append(tokens) diff --git a/reme4/reme.py b/reme4/reme.py index 437ef0cf..632bb2e2 100644 --- a/reme4/reme.py +++ b/reme4/reme.py @@ -1,3 +1,5 @@ +"""ReMe memory management application entry point.""" + import asyncio import sys @@ -12,6 +14,7 @@ class ReMe(Application): def main(): + """Parse CLI arguments and launch the appropriate mode.""" action, config = parse_args(sys.argv[1:]) if action == "start": reme = ReMe(**config) diff --git a/reme4/schema/emb_node.py b/reme4/schema/emb_node.py index d4191a26..2474a4e6 100644 --- a/reme4/schema/emb_node.py +++ b/reme4/schema/emb_node.py @@ -19,6 +19,7 @@ class EmbNode(BaseModel): @field_validator("embedding", mode="before") @classmethod def validate_embedding(cls, v): + """Coerce list/tuple to float16 ndarray.""" # Coerce list/tuple inputs into a float16 ndarray for compact storage. if v is None: return v @@ -26,6 +27,7 @@ class EmbNode(BaseModel): @field_serializer("embedding") def serialize_embedding(self, v: np.ndarray | None, _info): + """Serialize ndarray to a JSON-friendly list.""" # ndarray is not JSON-serializable; emit a plain list. if v is None: return None diff --git a/reme4/schema/file_link.py b/reme4/schema/file_link.py index 3d0b436f..b93cf1ca 100644 --- a/reme4/schema/file_link.py +++ b/reme4/schema/file_link.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, ConfigDict, Field class FileLink(BaseModel): + """A parsed wikilink with optional anchor and predicate.""" + model_config = ConfigDict(extra="forbid") path: str = Field( diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index 16df084f..2f8da442 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -2,13 +2,12 @@ import copy from abc import abstractmethod, ABC -from typing import TypeVar +from typing import TypeVar, TYPE_CHECKING from agentscope.formatter import FormatterBase from agentscope.model import ChatModelBase from agentscope.token import TokenCounterBase -from ..components import ApplicationContext from ..components.embedding import BaseEmbeddingModel from ..components.file_parser import BaseFileParser from ..components.file_store import BaseFileStore @@ -17,6 +16,9 @@ from ..components.runtime_context import RuntimeContext from ..enumeration import ComponentEnum from ..utils import get_logger +if TYPE_CHECKING: + from ..components import ApplicationContext + T = TypeVar("T") @@ -31,15 +33,15 @@ class BaseStep(ABC): return instance def __init__( - self, - name: str | None = None, - backend: str = "", - app_context: "ApplicationContext | None" = None, - language: str = "", - prompt_dict: dict[str, str] | None = None, - input_mapping: dict[str, str] | None = None, - output_mapping: dict[str, str] | None = None, - **kwargs, + self, + name: str | None = None, + backend: str = "", + app_context: "ApplicationContext | None" = None, + language: str = "", + prompt_dict: dict[str, str] | None = None, + input_mapping: dict[str, str] | None = None, + output_mapping: dict[str, str] | None = None, + **kwargs, ): super().__init__() self.name: str = name or self.__class__.__name__ @@ -85,32 +87,40 @@ class BaseStep(ABC): @property def as_llm(self) -> ChatModelBase: + """Return the chat model component.""" return self._resolve("as_llm", ChatModelBase, ComponentEnum.AS_LLM, "model") @property def as_llm_formatter(self) -> FormatterBase: + """Return the LLM formatter component.""" return self._resolve("as_llm_formatter", FormatterBase, ComponentEnum.AS_LLM_FORMATTER, "formatter") @property def as_token_counter(self) -> TokenCounterBase: + """Return the token counter component.""" return self._resolve("as_token_counter", TokenCounterBase, ComponentEnum.AS_TOKEN_COUNTER, "token_counter") @property def file_parser(self) -> BaseFileParser: + """Return the file parser component.""" return self._resolve("file_parser", BaseFileParser, ComponentEnum.FILE_PARSER) @property def file_store(self) -> BaseFileStore: + """Return the file store component.""" return self._resolve("file_store", BaseFileStore, ComponentEnum.FILE_STORE) @property def embedding(self) -> BaseEmbeddingModel: + """Return the embedding model component.""" return self._resolve("embedding", BaseEmbeddingModel, ComponentEnum.EMBEDDING_MODEL) def prompt_format(self, prompt_name: str, **kwargs) -> str: + """Format a named prompt template with the given kwargs.""" return self.prompt.prompt_format(prompt_name=prompt_name, **kwargs) def get_prompt(self, prompt_name: str) -> str: + """Return a named prompt template as-is.""" return self.prompt.get_prompt(prompt_name=prompt_name) def copy(self, **kwargs) -> "BaseStep":