ReMe/reme/utils/service_utils.py
jinliyl 2e87b7a52e
Some checks are pending
Pre-commit / run (ubuntu-latest) (push) Waiting to run
Tests ReMe / Unit Tests - py3.11 (push) Waiting to run
Tests ReMe / Unit Tests - py3.12 (push) Waiting to run
Tests ReMe / Unit Tests - py3.13 (push) Waiting to run
Windows Smoke / CLI smoke - py3.11 (push) Waiting to run
feat(core): add shell execution and runtime memory status (#344)
* feat(core): add shell command execution and memory status reporting

- Introduce ShellStep for executing shell commands with timeout support
- Add StatusStep to report memory estimates for stateful data components
- Register shell and status commands in default configuration
- Update documentation with new reme status and shell command capabilities
- Implement comprehensive unit tests for both new step types
- Add support for asynchronous command execution with proper error handling

* feat(config): add log_config option to suppress config loading logs

- Add log_config parameter to resolve_app_config function with default True
- Conditionally log config loading messages based on log_config flag
- Update reme.py and service_utils.py to use log_config=False for client calls
- Suppress config logging in user-facing contexts to avoid output pollution

refactor(shell): rename command parameter to cmd for clarity

- Change 'command' to 'cmd' in default.yaml configuration schema
- Rename 'timeout' to 'shell_timeout' to avoid parameter name collisions
- Update ShellStep to accept both legacy and new parameter names
- Maintain backward compatibility with existing command/timeout usage

test(shell): add comprehensive tests for shell step parameter handling

- Add test cases for new cmd and shell_timeout parameter names
- Verify legacy command and timeout parameters still work
- Test blank command rejection message updated to use cmd
- Create integration test for shell parameter payload passing

* fix(shell): ensure proper environment loading and process timeout handling

- Move load_env() call to execute before parse_args() in main function
- Add proper process group killing for timeout scenarios on POSIX systems
- Implement recursive child process termination on Windows for proper cleanup
- Change parameter name from 'timeout' to 'shell_timeout' in shell execution
- Remove support for legacy 'command' and 'timeout' parameter names
- Update test cases to verify new timeout behavior and parameter requirements
- Add comments explaining component size tracking implementation details
2026-07-14 16:31:41 +08:00

148 lines
5.5 KiB
Python

"""Service discovery utilities."""
import asyncio
import socket
import sys
import psutil
from ..constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT
async def find_reme(host: str, port: int) -> str:
"""Probe host:port. Returns 'reme', 'occupied', or 'free'."""
from ..components.client.http_client import HttpClient
try:
async with HttpClient(host=host, port=port, timeout=2.0) as client:
async for _ in client(action="health_check"):
break
return "reme"
except Exception:
pass
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.bind((host, port))
return "free"
except OSError:
return "occupied"
def _pid_on_port(port: int) -> int | None:
"""PID listening on TCP ``port``, or None. Cross-platform via psutil.
Iterates per-process rather than calling the system-wide
``psutil.net_connections()`` — the latter needs root on macOS, while
per-process connection enumeration works without elevation for
processes the current user owns (which reme's own server always is).
"""
for proc in psutil.process_iter(["pid"]):
try:
for conn in proc.net_connections(kind="tcp"):
if conn.status == psutil.CONN_LISTEN and conn.laddr and conn.laddr.port == port:
return proc.info["pid"]
except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess):
continue
return None
def _scan_reme_procs() -> list[tuple[int, str, int]]:
"""List running 'reme ... start' processes as (pid, host, port)."""
procs: list[tuple[int, str, int]] = []
for proc in psutil.process_iter(["pid", "cmdline"]):
try:
cmdline = proc.info["cmdline"] or []
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
# Match a `reme ... start` invocation (mirrors the old `pgrep -af`).
if "start" not in cmdline or not any("reme" in tok for tok in cmdline):
continue
host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
for t in cmdline:
if t.startswith("service.host="):
host = t.split("=", 1)[1]
elif t.startswith("service.port=") and t.split("=", 1)[1].isdigit():
port = int(t.split("=", 1)[1])
procs.append((proc.info["pid"], host, port))
return procs
def _reme_start_argv() -> list[list[str]]:
"""Return the `key=value` start-args of each running `reme ... start` process.
These cmdline tokens are the authoritative record of how the server was
*actually* launched — including ``service.*`` overrides that never touched
any config file — so replaying them reproduces the server's own config.
"""
argvs: list[list[str]] = []
for proc in psutil.process_iter(["cmdline"]):
try:
cmdline = proc.info["cmdline"] or []
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
if "start" not in cmdline or not any("reme" in tok for tok in cmdline):
continue
start_idx = cmdline.index("start")
argvs.append([t for t in cmdline[start_idx + 1 :] if "=" in t])
return argvs
def running_service_config() -> dict | None:
"""Resolve the ``service`` config of a running reme by replaying its start args.
Reads the live ``reme start ...`` process cmdline and re-runs the same
``resolve_app_config`` the server used, so the result matches the running
server's real backend/transport/host/port even when those were passed on the
command line and are absent from (or differ from) the on-disk config file.
Returns ``None`` when no running reme is found or its args can't be parsed.
"""
from ..config import parse_args, resolve_app_config
for argv in _reme_start_argv():
try:
_, kwargs = parse_args("start", *argv)
except ValueError:
continue
service = resolve_app_config(log_config=False, **kwargs).get("service")
if isinstance(service, dict):
return service
return None
async def locate_reme() -> tuple[str, int, int | None] | None:
"""Find a running reme: try default port, then scanned processes."""
if await find_reme(REME_DEFAULT_HOST, REME_DEFAULT_PORT) == "reme":
return REME_DEFAULT_HOST, REME_DEFAULT_PORT, _pid_on_port(REME_DEFAULT_PORT)
for pid, host, port in _scan_reme_procs():
if await find_reme(host, port) == "reme":
return host, port, pid
return None
def precheck_start(svc_config: dict | None) -> bool:
"""Pre-flight check for `start`: False if reme is up, exits 1 on port conflict."""
host = (svc_config or {}).get("host") or REME_DEFAULT_HOST
port = (svc_config or {}).get("port") or REME_DEFAULT_PORT
port = int(port)
status = asyncio.run(find_reme(host, port))
if status == "reme":
print(f"reme already running at {host}:{port}")
return False
if status == "occupied":
print(
f"port {port} occupied. Start on another port: reme start service.port=<other_port>",
file=sys.stderr,
)
sys.exit(1)
return True
def cli_find_reme() -> None:
"""Handle `reme find_reme`: print HOST/PORT/PID or a hint to start reme."""
found = asyncio.run(locate_reme())
if not found:
print("reme not started. Try: reme start", file=sys.stderr)
sys.exit(1)
host, port, pid = found
print(f"HOST={host} PORT={port} PID={pid or 'unknown'}")