mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-13 04:01:35 +00:00
e4c381c7b1
benchmark.yaml's python_execute job advertised a `timeout` parameter, but `timeout` is one of the CLI's client-selection kwargs (reme/reme.py pops it before the payload is sent), so `reme python_execute code=... timeout=300` never delivered the timeout to the job. The value silently became the HTTP client's request timeout while the subprocess kept the 60s default, so benchmark code that needs longer was killed at 60s with no hint why. HTTP and MCP callers were unaffected. Rename the parameter to `python_timeout`, mirroring how the shell job already avoids the same collision with `shell_timeout`. The step prefers the new name and still accepts the legacy `timeout` context key so existing benchmark configs keep working. A new guard test fails if any job in reme/config/*.yaml or a plugin plugin.yaml declares a client-reserved parameter name again.
434 lines
14 KiB
Python
434 lines
14 KiB
Python
"""Tests for the ReMe CLI entry helpers."""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from reme.components.service import cli_service
|
|
from reme.components.service.cli_service import CliService
|
|
from reme import reme as reme_module
|
|
from reme.components.base_component import ComponentMixin
|
|
from reme.components.component_registry import create_application_registry
|
|
from reme.enumeration import ComponentEnum
|
|
from reme.plugin import Backend, Plugin, PluginManager, PluginRuntime
|
|
|
|
|
|
def _recording_client(seen, output="ok", base=object):
|
|
"""Build an async client stub that records construction and calls."""
|
|
|
|
class RecordingClient(base):
|
|
"""Async client stub backed by a shared call record."""
|
|
|
|
def __init__(self, **kwargs):
|
|
seen["client_kwargs"] = kwargs
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
return None
|
|
|
|
async def __call__(self, action: str, **kwargs):
|
|
seen["action"] = action
|
|
seen["payload"] = kwargs
|
|
yield output
|
|
|
|
return RecordingClient
|
|
|
|
|
|
def _set_client_backend(monkeypatch, client_cls):
|
|
monkeypatch.setattr(
|
|
reme_module,
|
|
"resolve_plugin_runtime",
|
|
lambda config: PluginRuntime(
|
|
config=dict(config),
|
|
registry=SimpleNamespace(get=lambda component_type, backend: client_cls),
|
|
),
|
|
)
|
|
|
|
|
|
def test_package_import_does_not_load_optional_core_dependencies():
|
|
"""The base package leaves optional core dependencies unloaded."""
|
|
script = """
|
|
import importlib.abc
|
|
|
|
|
|
class BlockOptionalCoreDependencies(importlib.abc.MetaPathFinder):
|
|
def find_spec(self, fullname, path, target=None):
|
|
blocked = (
|
|
"claude_agent_sdk",
|
|
"dingtalk_stream",
|
|
"openai_codex",
|
|
"faiss",
|
|
"jieba",
|
|
"rjieba",
|
|
"neo4j",
|
|
"networkx",
|
|
"pypdf",
|
|
"polars",
|
|
"tushare",
|
|
)
|
|
if any(fullname == name or fullname.startswith(f"{name}.") for name in blocked):
|
|
raise AssertionError(f"eagerly imported optional core dependency: {fullname}")
|
|
return None
|
|
|
|
|
|
import sys
|
|
|
|
sys.meta_path.insert(0, BlockOptionalCoreDependencies())
|
|
import reme
|
|
"""
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", script],
|
|
cwd=Path(__file__).resolve().parents[2],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def test_main_loads_env_before_calling_server(monkeypatch):
|
|
"""Client actions can resolve connection settings from the local .env."""
|
|
events = []
|
|
|
|
main_globals = reme_module.main.__globals__
|
|
monkeypatch.setattr("sys.argv", ["reme", "shell", "cmd=pwd"])
|
|
monkeypatch.setitem(main_globals, "load_env", lambda: events.append("load_env"))
|
|
|
|
async def fake_call_server(action, **kwargs):
|
|
events.append(("call_server", action, kwargs))
|
|
|
|
monkeypatch.setitem(main_globals, "call_server", fake_call_server)
|
|
|
|
reme_module.main()
|
|
|
|
assert events == ["load_env", ("call_server", "shell", {"cmd": "pwd"})]
|
|
|
|
|
|
def test_main_saves_loaded_environment_in_start_config(monkeypatch):
|
|
"""The startup config keeps the environment captured by the single global load."""
|
|
observed = {}
|
|
|
|
class FakeReMe:
|
|
"""Capture the fully resolved application configuration."""
|
|
|
|
def __init__(self, **kwargs):
|
|
"""Record the application startup configuration."""
|
|
observed["config"] = kwargs
|
|
|
|
def run_app(self):
|
|
"""Record that application startup continued."""
|
|
observed["ran"] = True
|
|
|
|
main_globals = reme_module.main.__globals__
|
|
monkeypatch.setattr("sys.argv", ["reme", "start"])
|
|
monkeypatch.setitem(main_globals, "load_env", lambda: {"TOOL_ENV": "configured"})
|
|
monkeypatch.setitem(main_globals, "prepare_start_config", lambda _kwargs: {"service": {"backend": "cli"}})
|
|
monkeypatch.setitem(main_globals, "ReMe", FakeReMe)
|
|
|
|
reme_module.main()
|
|
|
|
assert observed == {
|
|
"config": {
|
|
"service": {"backend": "cli"},
|
|
"environment": {"TOOL_ENV": "configured"},
|
|
},
|
|
"ran": True,
|
|
}
|
|
|
|
|
|
def test_prepare_start_config_moves_unknown_start_args_to_job_args(monkeypatch):
|
|
"""``reme start job=...`` is translated into a one-shot cli service config."""
|
|
|
|
monkeypatch.setattr(
|
|
cli_service,
|
|
"resolve_app_config",
|
|
lambda **kwargs: {
|
|
**kwargs,
|
|
"service": {"backend": "http", "host": "127.0.0.1"},
|
|
},
|
|
)
|
|
|
|
cfg = cli_service.prepare_start_config(
|
|
{
|
|
"config": "jinli_lme",
|
|
"workspace_dir": "/tmp/reme",
|
|
"job": "search",
|
|
"query": "hello",
|
|
"limit": 3,
|
|
},
|
|
)
|
|
|
|
assert cfg["config"] == "jinli_lme"
|
|
assert cfg["workspace_dir"] == "/tmp/reme"
|
|
assert cfg["enable_logo"] is False
|
|
assert cfg["log_to_console"] is False
|
|
assert cfg["service"] == {
|
|
"backend": "cli",
|
|
"host": "127.0.0.1",
|
|
"job": "search",
|
|
"job_args": {"query": "hello", "limit": 3},
|
|
}
|
|
|
|
|
|
def test_should_precheck_start_skips_cli_service():
|
|
"""CLI service is local execution and should not run port prechecks."""
|
|
assert cli_service.should_precheck_start({"service": {"backend": "cli"}}) is False
|
|
assert cli_service.should_precheck_start({"service": {"backend": "http"}}) is True
|
|
|
|
|
|
def test_cli_service_runs_configured_job_and_closes_app(capsys):
|
|
"""CLI service runs one local job through app lifecycle and prints its answer."""
|
|
events = []
|
|
|
|
class FakeApp:
|
|
"""Minimal app stub for exercising CliService lifecycle."""
|
|
|
|
async def start(self):
|
|
"""Record app startup."""
|
|
events.append("start")
|
|
|
|
async def close(self):
|
|
"""Record app shutdown."""
|
|
events.append("close")
|
|
|
|
async def run_job(self, name, **kwargs):
|
|
"""Record job execution and return a successful response."""
|
|
events.append(("run_job", name, kwargs))
|
|
return SimpleNamespace(answer="found it", success=True, metadata={"hits": 1})
|
|
|
|
service = CliService(job="search", job_args={"query": "hello"})
|
|
|
|
service.start_service(FakeApp())
|
|
|
|
assert events == [
|
|
"start",
|
|
("run_job", "search", {"query": "hello"}),
|
|
"close",
|
|
]
|
|
assert capsys.readouterr().out == "found it\n"
|
|
|
|
|
|
def test_cli_service_can_print_metadata_from_service_config(capsys):
|
|
"""service.show_metadata controls optional CLI metadata output."""
|
|
|
|
class FakeApp:
|
|
"""Minimal app stub for exercising metadata output."""
|
|
|
|
async def start(self):
|
|
"""No-op app startup."""
|
|
|
|
async def close(self):
|
|
"""No-op app shutdown."""
|
|
|
|
async def run_job(self, _name, **_kwargs):
|
|
"""Return a successful response with metadata."""
|
|
return SimpleNamespace(answer="found it", success=True, metadata={"hits": 1})
|
|
|
|
service = CliService(job="search", show_metadata=True)
|
|
|
|
service.start_service(FakeApp())
|
|
|
|
assert capsys.readouterr().out == 'found it\n{"hits": 1}\n'
|
|
|
|
|
|
def test_cli_service_exits_nonzero_on_failed_response(capsys):
|
|
"""Failed local CLI jobs write to stderr and produce a failing process status."""
|
|
events = []
|
|
|
|
class FakeApp:
|
|
"""Minimal app stub for exercising failure handling."""
|
|
|
|
async def start(self):
|
|
"""Record app startup."""
|
|
events.append("start")
|
|
|
|
async def close(self):
|
|
"""Record app shutdown."""
|
|
events.append("close")
|
|
|
|
async def run_job(self, name, **kwargs):
|
|
"""Record job execution and return a failed response."""
|
|
events.append(("run_job", name, kwargs))
|
|
return SimpleNamespace(answer="boom", success=False, metadata={})
|
|
|
|
service = CliService(job="search", job_args={"query": "hello"})
|
|
|
|
with pytest.raises(SystemExit) as exc_info:
|
|
service.start_service(FakeApp())
|
|
|
|
assert exc_info.value.code == 1
|
|
assert events == [
|
|
"start",
|
|
("run_job", "search", {"query": "hello"}),
|
|
"close",
|
|
]
|
|
captured = capsys.readouterr()
|
|
assert captured.out == ""
|
|
assert captured.err == "boom\n"
|
|
|
|
|
|
def test_call_server_passes_client_kwargs_to_client(monkeypatch, capsys):
|
|
"""CLI helper forwards connection options to the selected client."""
|
|
seen = {}
|
|
_set_client_backend(monkeypatch, _recording_client(seen))
|
|
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
|
|
|
async def run():
|
|
await reme_module.call_server(
|
|
"search",
|
|
backend="http",
|
|
host="127.0.0.2",
|
|
port=2444,
|
|
timeout=1.5,
|
|
query="hello",
|
|
)
|
|
|
|
asyncio.run(run())
|
|
|
|
assert seen["client_kwargs"] == {"host": "127.0.0.2", "port": 2444, "timeout": 1.5}
|
|
assert seen["action"] == "search"
|
|
assert seen["payload"] == {"query": "hello"}
|
|
assert capsys.readouterr().out == "ok\n"
|
|
|
|
|
|
def test_call_server_treats_show_metadata_as_client_kwarg(monkeypatch, capsys):
|
|
"""show_metadata controls client display and is not sent as a tool argument."""
|
|
seen = {}
|
|
_set_client_backend(monkeypatch, _recording_client(seen))
|
|
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
|
|
|
async def run():
|
|
await reme_module.call_server("version", backend="http", show_metadata=True)
|
|
|
|
asyncio.run(run())
|
|
|
|
assert seen["client_kwargs"] == {"show_metadata": True}
|
|
assert seen["action"] == "version"
|
|
assert seen["payload"] == {}
|
|
assert capsys.readouterr().out == "ok\n"
|
|
|
|
|
|
def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys):
|
|
"""Shell-specific parameter names do not collide with client options."""
|
|
seen = {}
|
|
_set_client_backend(monkeypatch, _recording_client(seen))
|
|
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
|
|
|
async def run():
|
|
await reme_module.call_server("shell", backend="http", cmd="ls", shell_timeout=5)
|
|
|
|
asyncio.run(run())
|
|
|
|
assert seen["action"] == "shell"
|
|
assert seen["payload"] == {"cmd": "ls", "shell_timeout": 5}
|
|
assert capsys.readouterr().out == "ok\n"
|
|
|
|
|
|
def test_call_server_passes_python_execute_parameters_as_payload(monkeypatch, capsys):
|
|
"""python_execute's timeout parameter avoids the client-side timeout option."""
|
|
seen = {}
|
|
_set_client_backend(monkeypatch, _recording_client(seen))
|
|
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
|
|
|
async def run():
|
|
await reme_module.call_server(
|
|
"python_execute",
|
|
backend="http",
|
|
code="print(1)",
|
|
python_timeout=5,
|
|
timeout=1.5,
|
|
)
|
|
|
|
asyncio.run(run())
|
|
|
|
assert seen["action"] == "python_execute"
|
|
assert seen["payload"] == {"code": "print(1)", "python_timeout": 5}
|
|
assert seen["client_kwargs"] == {"timeout": 1.5}
|
|
assert capsys.readouterr().out == "ok\n"
|
|
|
|
|
|
def test_no_job_parameter_collides_with_client_kwargs():
|
|
"""A job parameter named like a client option could never be set through the CLI."""
|
|
reserved = set(reme_module._CLIENT_KWARGS) | {"backend"} # pylint: disable=protected-access
|
|
repo_root = Path(reme_module.__file__).resolve().parents[1]
|
|
configs = sorted((repo_root / "reme" / "config").glob("*.yaml"))
|
|
configs += sorted(repo_root.glob("plugins/*/src/*/plugin.yaml"))
|
|
for config_path in configs:
|
|
jobs = yaml.safe_load(config_path.read_text(encoding="utf-8")).get("jobs") or {}
|
|
for job_name, job in jobs.items():
|
|
properties = ((job or {}).get("parameters") or {}).get("properties") or {}
|
|
clash = sorted(set(properties) & reserved)
|
|
assert not clash, (
|
|
f"{config_path.relative_to(repo_root)} job '{job_name}' declares client-reserved "
|
|
f"parameters: {clash}"
|
|
)
|
|
|
|
|
|
def test_call_server_uses_running_plugins_and_their_service_defaults(monkeypatch, capsys):
|
|
"""A bare client call can load the Client backend enabled by the running app."""
|
|
seen = {}
|
|
plugin_client = _recording_client(seen, output="plugin-ok", base=ComponentMixin)
|
|
plugin_client.component_type = ComponentEnum.CLIENT
|
|
|
|
manager = PluginManager(
|
|
[
|
|
Plugin(
|
|
name="example",
|
|
backends=(Backend("plugin-client", plugin_client),),
|
|
config={
|
|
"service": {
|
|
"backend": "plugin-client",
|
|
"host": "127.0.0.9",
|
|
"port": 9911,
|
|
},
|
|
},
|
|
),
|
|
],
|
|
)
|
|
monkeypatch.setattr(
|
|
reme_module,
|
|
"resolve_app_config",
|
|
lambda **_kwargs: {"service": {"backend": "http"}},
|
|
)
|
|
monkeypatch.setattr(reme_module, "running_app_config", lambda: {"plugins": ["example"]})
|
|
|
|
def resolve_runtime(config):
|
|
registry = create_application_registry()
|
|
manager.register(registry)
|
|
return PluginRuntime(config=manager.merge_config(config), registry=registry)
|
|
|
|
monkeypatch.setattr(reme_module, "resolve_plugin_runtime", resolve_runtime)
|
|
|
|
asyncio.run(reme_module.call_server("search", query="hello"))
|
|
|
|
assert seen["client_kwargs"]["host"] == "127.0.0.9"
|
|
assert seen["client_kwargs"]["port"] == 9911
|
|
assert seen["action"] == "search"
|
|
assert seen["payload"] == {"query": "hello"}
|
|
assert capsys.readouterr().out == "plugin-ok\n"
|
|
|
|
|
|
def test_call_server_skips_local_fallback_when_server_is_running(monkeypatch, capsys):
|
|
"""A usable running config prevents eager parsing of the local fallback."""
|
|
seen = {}
|
|
_set_client_backend(monkeypatch, _recording_client(seen))
|
|
monkeypatch.setattr(reme_module, "running_app_config", lambda: {"service": {"backend": "http"}})
|
|
|
|
def fail_local_resolution(**_kwargs):
|
|
raise AssertionError("local fallback should not be resolved")
|
|
|
|
monkeypatch.setattr(reme_module, "resolve_app_config", fail_local_resolution)
|
|
|
|
asyncio.run(reme_module.call_server("version"))
|
|
|
|
assert seen["action"] == "version"
|
|
assert seen["payload"] == {}
|
|
assert capsys.readouterr().out == "ok\n"
|