From 8a7ec516c81cfa0b47e4adc4873abc5d5b8f2fd3 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Sat, 16 May 2026 01:02:18 +0800 Subject: [PATCH] up --- pyproject.toml | 24 ++-- reme4/components/client/__init__.py | 3 +- reme4/components/client/base_client.py | 4 +- reme4/components/client/http_client.py | 7 +- reme4/components/client/mcp_client.py | 115 ++++++++++++++++++ .../components/file_store/local_file_store.py | 4 +- .../file_watcher/base_file_watcher.py | 7 +- .../file_watcher/lite_file_watcher.py | 31 +++-- 8 files changed, 158 insertions(+), 37 deletions(-) create mode 100644 reme4/components/client/mcp_client.py diff --git a/pyproject.toml b/pyproject.toml index 11177ffd..6bbbb74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,9 +33,7 @@ classifiers = [ keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http", "reme", "personal"] dependencies = [ - "sqlite-vec>=0.1.6", - "prompt_toolkit>=3.0.52", - "rich>=14.2.0", + "aiofiles>=24.1.0", "asyncpg>=0.31.0", "chromadb>=1.3.5", "dashscope>=1.25.1", @@ -43,23 +41,28 @@ dependencies = [ "fastapi>=0.121.3", "fastmcp>=2.14.1", "httpx>=0.28.1", + "jieba>=0.42.1", "loguru>=0.7.3", "mcp>=1.25.0", + "networkx>=3.4", "numpy>=2.2.6", "openai>=2.8.1", "pandas>=2.3.3", + "prompt_toolkit>=3.0.52", "pydantic>=2.12.4", "pyobvector>=0.1.20", + "pyyaml>=6.0.3", + "qdrant-client>=1.16.0", + "rich>=14.2.0", + "sqlite-vec>=0.1.6", # pyobvector imports Expression from sqlglot; removed from sqlglot 30+ top-level API "sqlglot>=25,<30", - "qdrant-client>=1.16.0", "tavily-python>=0.7.13", "tiktoken>=0.12.0", "tqdm>=4.67.1", "transformers>=4.57.3", "uvicorn>=0.40.0", "watchfiles>=1.1.1", - "pyyaml>=6.0.3", ] [project.optional-dependencies] @@ -85,14 +88,14 @@ litellm = [ "litellm==1.80.0", ] -light = [ +core = [ "agentscope==1.0.18", "flowllm[reme]>=0.2.0.10", ] [tool.setuptools.packages.find] where = ["."] -include = ["reme_ai*", "reme*"] +include = ["reme_ai*", "reme*", "reme4*"] exclude = ["test*", "cookbook*", "doc*", "library*", "dist*"] [tool.setuptools.package-data] @@ -108,6 +111,12 @@ reme = [ "**/*.json", ] +reme4 = [ + "**/*.yaml", + "**/*.py", + "**/*.json", +] + [tool.setuptools.dynamic] version = { attr = "reme.__version__" } @@ -120,6 +129,7 @@ Repository = "https://github.com/agentscope-ai/ReMe" reme = "reme_ai.main:main" reme2 = "reme.reme:main" remecli = "reme.reme_cli:main" +reme4 = "reme4.reme:main" [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" diff --git a/reme4/components/client/__init__.py b/reme4/components/client/__init__.py index e5a1df0a..5c212bac 100644 --- a/reme4/components/client/__init__.py +++ b/reme4/components/client/__init__.py @@ -2,5 +2,6 @@ from .base_client import BaseClient from .http_client import HttpClient +from .mcp_client import MCPClient -__all__ = ["BaseClient", "HttpClient"] +__all__ = ["BaseClient", "HttpClient", "MCPClient"] diff --git a/reme4/components/client/base_client.py b/reme4/components/client/base_client.py index 42bae599..4ca93298 100644 --- a/reme4/components/client/base_client.py +++ b/reme4/components/client/base_client.py @@ -22,5 +22,5 @@ class BaseClient(BaseComponent): """Close the client and release resources.""" @abstractmethod - async def __call__(self) -> dict: - """Execute the configured action and return the response.""" + async def __call__(self) -> str: + """Execute the configured action""" diff --git a/reme4/components/client/http_client.py b/reme4/components/client/http_client.py index 7a11413a..cbcc7ec9 100644 --- a/reme4/components/client/http_client.py +++ b/reme4/components/client/http_client.py @@ -46,14 +46,17 @@ class HttpClient(BaseClient): if self.client is None: self.client = httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) - async def __call__(self) -> dict: + async def __call__(self) -> str: """Send POST request to the configured action endpoint.""" if self.client is None: raise RuntimeError("Client not initialized. Call _start() first.") response = await self.client.post(f"/{self.action}", json=self.kwargs) response.raise_for_status() - return response.json() + result = response.json() + if "answer" in result: + return str(result["answer"]) + return str(result) async def _close(self) -> None: """Close the HTTP client.""" diff --git a/reme4/components/client/mcp_client.py b/reme4/components/client/mcp_client.py new file mode 100644 index 00000000..ce74d690 --- /dev/null +++ b/reme4/components/client/mcp_client.py @@ -0,0 +1,115 @@ +"""MCP client for ReMe services.""" + +import json +import os +from typing import Any + +from fastmcp import Client +from fastmcp.client import SSETransport, StdioTransport, StreamableHttpTransport +from fastmcp.client.client import CallToolResult + +from .base_client import BaseClient +from ..component_registry import R +from ...constants import REME_SERVICE_INFO, REME_DEFAULT_HOST, REME_DEFAULT_PORT + +_TRANSPORT_MAP = { + "sse": SSETransport, + "stdio": StdioTransport, + "streamable-http": StreamableHttpTransport, +} + + +@R.register("mcp") +class MCPClient(BaseClient): + """MCP client that communicates with ReMe MCP service via fastmcp.Client. + + Usage: + # SSE (default) + client = MCPClient(action="my_tool", host="localhost", port=8000, query="hello") + async with client: + result = await client() + + # Streamable HTTP + client = MCPClient(action="my_tool", transport="streamable-http", host="localhost", port=8000) + + # Stdio + client = MCPClient(action="my_tool", transport="stdio", command="python", args=["server.py"]) + + # Custom transport object + from fastmcp.client import SSETransport + client = MCPClient(action="my_tool", transport=SSETransport(url="http://host:port/sse")) + """ + + def __init__( + self, + action: str, + transport: str | Any = "sse", + host: str | None = None, + port: int | None = None, + timeout: float = 30.0, + **kwargs, + ): + super().__init__(**kwargs) + + if isinstance(transport, str) and transport not in _TRANSPORT_MAP: + raise ValueError(f"Unknown transport: {transport!r}, expected one of {list(_TRANSPORT_MAP)}") + + if isinstance(transport, str) and transport != "stdio": + if not (host and port): + if service_info := os.environ.get(REME_SERVICE_INFO): + try: + data = json.loads(service_info) + host = data["host"] + port = data["port"] + except Exception: + self.logger.warning(f"Invalid service info: {service_info}") + host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT + else: + host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT + self.host = host + self.port = port + + self.action = action + self.transport = transport + self.timeout = timeout + + def _build_transport(self): + if not isinstance(self.transport, str): + return self.transport + + cls = _TRANSPORT_MAP[self.transport] + + if self.transport == "stdio": + command = self.kwargs.pop("command", "") + args = self.kwargs.pop("args", []) + return cls(command=command, args=args) + + path = "/sse" if self.transport == "sse" else "/mcp" + url = f"http://{self.host}:{self.port}{path}" + return cls(url=url) + + # pylint: disable=unnecessary-dunder-call + async def _start(self) -> None: + if self.client is None: + self.client = Client(self._build_transport(), timeout=self.timeout) + await self.client.__aenter__() + + async def __call__(self) -> str: + if self.client is None: + raise RuntimeError("Client not initialized. Call _start() first.") + + result: CallToolResult = await self.client.call_tool(self.action, self.kwargs) + return self._extract_text(result) + + # pylint: disable=unnecessary-dunder-call + async def _close(self) -> None: + if self.client is not None: + await self.client.__aexit__(None, None, None) + self.client = None + + @staticmethod + def _extract_text(result: CallToolResult) -> str: + for block in result.content: + if hasattr(block, "text"): + return block.text + return str(result.content) diff --git a/reme4/components/file_store/local_file_store.py b/reme4/components/file_store/local_file_store.py index ebda9562..f65b7fd7 100644 --- a/reme4/components/file_store/local_file_store.py +++ b/reme4/components/file_store/local_file_store.py @@ -6,9 +6,7 @@ import numpy as np from .base_file_store import BaseFileStore from ..component_registry import R from ...schema import FileChunk, FileNode -from ...utils import batch_cosine_similarity, get_logger - -logger = get_logger() +from ...utils import batch_cosine_similarity @R.register("local") diff --git a/reme4/components/file_watcher/base_file_watcher.py b/reme4/components/file_watcher/base_file_watcher.py index 79e111dd..60142692 100644 --- a/reme4/components/file_watcher/base_file_watcher.py +++ b/reme4/components/file_watcher/base_file_watcher.py @@ -10,9 +10,6 @@ from ..base_component import BaseComponent from ..file_parser import BaseFileParser from ..file_store import BaseFileStore from ...enumeration import ComponentEnum -from ...utils import get_logger - -logger = get_logger() class BaseFileWatcher(BaseComponent): @@ -50,7 +47,7 @@ class BaseFileWatcher(BaseComponent): async def _start(self): self._stop_event = asyncio.Event() self._background_task = asyncio.create_task(self._background_run()) - logger.info(f"Started watching: {self.watch_paths}") + self.logger.info(f"Started watching: {self.watch_paths}") async def _background_run(self): """Sync store then enter watch loop.""" @@ -61,7 +58,7 @@ class BaseFileWatcher(BaseComponent): self._stop_event.set() if self._background_task: await self._background_task - logger.info("Stopped watching") + self.logger.info("Stopped watching") def watch_filter(self, _change: Change, path: str) -> bool: """Return True if the file suffix matches the filter list.""" diff --git a/reme4/components/file_watcher/lite_file_watcher.py b/reme4/components/file_watcher/lite_file_watcher.py index 633a1e82..df5370e6 100644 --- a/reme4/components/file_watcher/lite_file_watcher.py +++ b/reme4/components/file_watcher/lite_file_watcher.py @@ -8,9 +8,6 @@ from watchfiles import Change, awatch from .base_file_watcher import BaseFileWatcher from ..component_registry import R from ...schema import FileChunk, FileNode -from ...utils import get_logger - -logger = get_logger() @R.register("lite") @@ -26,22 +23,22 @@ class LiteFileWatcher(BaseFileWatcher): async def watch_loop(self): if not self.watch_paths: - logger.warning("No watch paths specified") + self.logger.warning("No watch paths specified") return while not self._stop_event.is_set(): valid_paths = [p for p in self.watch_paths if p.exists()] if not valid_paths: - logger.warning(f"No valid paths, retrying in {self._retry_interval}s...") + self.logger.warning(f"No valid paths, retrying in {self._retry_interval}s...") await self._interruptible_sleep() continue invalid = set(self.watch_paths) - set(valid_paths) if invalid: - logger.warning(f"Skipping invalid paths: {invalid}") + self.logger.warning(f"Skipping invalid paths: {invalid}") try: - logger.info(f"Watching: {valid_paths}") + self.logger.info(f"Watching: {valid_paths}") async for changes in awatch( *valid_paths, watch_filter=self.watch_filter, @@ -55,7 +52,7 @@ class LiteFileWatcher(BaseFileWatcher): break await self._dispatch_changes(changes) except Exception: - logger.exception(f"Watch error, retrying in {self._retry_interval}s...") + self.logger.exception(f"Watch error, retrying in {self._retry_interval}s...") if not self._stop_event.is_set(): await self._interruptible_sleep() @@ -65,13 +62,13 @@ class LiteFileWatcher(BaseFileWatcher): modified = [Path(p) for c, p in changes if c == Change.modified] deleted = [Path(p) for c, p in changes if c == Change.deleted] if added: - logger.info(f"Detected {len(added)} added file(s)") + self.logger.info(f"Detected {len(added)} added file(s)") await self.on_added(added) if modified: - logger.info(f"Detected {len(modified)} modified file(s)") + self.logger.info(f"Detected {len(modified)} modified file(s)") await self.on_modified(modified) if deleted: - logger.info(f"Detected {len(deleted)} deleted file(s)") + self.logger.info(f"Detected {len(deleted)} deleted file(s)") await self.on_deleted(deleted) async def update_store(self): @@ -88,16 +85,16 @@ class LiteFileWatcher(BaseFileWatcher): to_modify = [p for p in existing_keys & indexed_keys if existing[p] != indexed[p]] if to_modify: - logger.info(f"Updating {len(to_modify)} modified file(s)") + self.logger.info(f"Updating {len(to_modify)} modified file(s)") await self.on_modified([Path(p) for p in to_modify]) if to_delete: - logger.info(f"Removing {len(to_delete)} deleted file(s)") + self.logger.info(f"Removing {len(to_delete)} deleted file(s)") await self.on_deleted([Path(p) for p in to_delete]) if to_add: - logger.info(f"Indexing {len(to_add)} new file(s)") + self.logger.info(f"Indexing {len(to_add)} new file(s)") await self.on_added([Path(p) for p in to_add]) if not to_modify and not to_delete and not to_add: - logger.info("Store is up to date") + self.logger.info("Store is up to date") async def _parse_and_upsert(self, paths: list[Path], action: str): """Parse files and upsert into store. Shared by on_added / on_modified.""" @@ -107,7 +104,7 @@ class LiteFileWatcher(BaseFileWatcher): parsed: list[tuple[FileNode, list[FileChunk]]] = [] for p in paths: if p.is_file(): - logger.info(f"{action} file: {p}") + self.logger.info(f"{action} file: {p}") parsed.append(await self.file_parser.parse(p)) if parsed: file_paths = [str(p) for p in paths if p.is_file()] @@ -126,5 +123,5 @@ class LiteFileWatcher(BaseFileWatcher): if self.file_store is None: raise RuntimeError("file_store is not initialized!") paths = [path] if isinstance(path, Path) else path - logger.info(f"Deleting {len(paths)} file(s)") + self.logger.info(f"Deleting {len(paths)} file(s)") await self.file_store.delete_by_path([str(p) for p in paths])