mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-07 08:26:06 +00:00
* feat(service): add CLI service for local job execution - Introduce CliService to execute single jobs locally without serving ports - Add prepare_start_config and should_precheck_start functions for CLI job setup - Update reme start command to use CLI service when job argument is provided - Change default service backend from http to cli in jinli_lme config - Modify SearchStep to use constants and rename configuration parameters - Add unit tests for CLI service functionality and configuration handling - Update file extension support to include json format in addition to md and jsonl * feat(search): add BM25 and vector search steps with configuration updates - Add Bm25SearchStep and VectorSearchStep classes with tool context deduplication - Register new search step components in index module - Update configuration to use separate vector_search and bm25_search endpoints - Modify LLM models from qwen3.7-plus/glm-5.1 to glm-5.2 variants - Adjust search parameters and remove hybrid search implementation - Configure embedding store as default in storage settings - Remove auto-memory and file catalog configurations - Update watch directories from multiple paths to session_dir only * feat(agent): add tool result offloading and workspace management - Add tool_results_dir configuration option for offloaded tool results storage - Implement ToolResultOffloadMiddleware to persist large tool results to files - Create WorkspaceBackend to standardize file operations across tools - Add configurable builtin tools selection with sequential execution option - Integrate middleware support for agent wrapper with offloading capability - Update application initialization to create tool results directory - Add safety mechanisms for filesystem operations with sanitized filenames - Enhance agent wrapper with configurable working directory handling - Upgrade agentscope dependency to version 2.0.4 for improved features # Conflicts: # reme/application.py * feat(benchmark): add LongMemEval agentic search and result management - Introduce AgenticAnswerStep for agent-based history search - Add LmePrepareJudgeStep and LmeSaveResultStep for evaluation pipeline - Implement AddDraftStep and ReadAllDraftStep for evidence accumulation - Update configuration with new agent wrapper and search parameters - Add comparison script for analyzing agent run differences - Include documentation for LongMemEval failure analysis - Enhance tool result offloading with skip options - Modify search defaults and indexing behavior * feat(agent): implement tool result offloading with system reminders - Added tool_result_offload_message parameter to agent wrapper reply method - Implemented configurable reminder template for offloaded tool results - Created system reminder messages when tool results are offloaded to files - Added Chinese user message template for agentic answer step - Updated tool result offloading middleware to use custom reminder templates - Enhanced agentic answer instructions to handle long tool results via draft storage * feat(scripts): add LongMemEval results summarization tool - Create summarize_lme_results.py script to analyze result JSON files - Implement command line interface with answer id and dataset root options - Add support for specifying index range with start and end parameters - Include option to show failure details and non-successful completions - Calculate completion statistics and accuracy metrics - Display detailed breakdown of yes/no/other judgements - Handle missing and unreadable result files gracefully - Format output with percentages and comprehensive summary statistics * feat(summarize_lme_results): add question type breakdown to result summary - Import defaultdict from collections module - Add by_type dictionary to track statistics by question type - Count completed, yes, no, and other responses for each question type - Display detailed breakdown table showing accuracy by question type - Include question type column when processing judgements - Print comprehensive summary with question type distribution - Calculate and display accuracy percentage for each question type category * feat(lme): switch to qwen3.7-max model and add shuffle functionality - Changed default LLM model from glm-5.1 to qwen3.7-max in jinli_lme.yaml - Added random module import for shuffle functionality - Implemented --shuffle argument with BooleanOptionalAction for dataset shuffling - Added --seed argument to control random seed for reproducible shuffling - Applied random shuffle to dataset indices when shuffle is enabled - Added console output showing shuffle operation and seed information * fix(cli): set default random seed for shuffle functionality - Changed default seed value from None to 42 for consistent shuffling behavior - Ensures reproducible results when using shuffle option without explicit seed - Maintains backward compatibility while providing deterministic defaults * refactor(benchmark): update agentic answer guidelines for grounding - Updated English instruction to emphasize strict grounding in retrieved context - Modified Chinese instruction to stress evidence-based responses without inference - Removed redundant conciseness requirement in both language versions - Enhanced clarity on proper use of draft saving and retrieval mechanisms - Strengthened emphasis against hallucination of unsupported facts * refactor(benchmark): update agentic search instructions and configuration - Replace separate vector_search and bm25_search with unified search tool - Update agent instructions to use single search tool with multiple strategies - Simplify Chinese instructions for search methodology - Add comprehensive search tool configuration with hybrid vector/BM25 capabilities - Increase model retry attempts from 1 to 3 for better reliability - Remove redundant tool references from job_tools list * feat(search): add configurable search limit with environment variable support - Remove hardcoded limit and min_score parameters from config schema - Increase LLM context size from 200000 to 1000000 - Add REME_SEARCH_LIMIT environment variable support for search configuration - Implement command line argument --search-limit to override default search limit - Add input validation to ensure search limit is positive - Modify subprocess execution to pass environment variables - Update search step to use dynamic default limit from environment or fallback to 5 * refactor(benchmark): remove agentic answer step and related configurations - Removed AgenticAnswerStep class and its registration - Deleted agentic_answer.yaml prompt configuration file - Removed agentic answer related job definitions from jinli_lme.yaml - Cleaned up tool result offloading middleware implementation - Removed tool_results_dir configuration field from application config - Deleted comparison and analysis scripts for agent runs - Removed agentic answer step from LME init module exports - Updated agent wrapper to remove tool result offloading functionality - Removed unused imports and dependencies in agent wrapper module * refactor(benchmark): remove unused LME result processing components - Removed LmePrepareJudgeStep and LmeSaveResultStep classes from benchmark module - Cleaned up imports and exports in lme module initialization - Removed unused middleware configuration from agent wrapper - Deleted obsolete result.py file containing deprecated result processing logic - Simplified agent instantiation by removing middleware parameter - Updated import statements to reflect removed dependencies * refactor(index): remove unused search steps and update imports - Remove Bm25SearchStep and VectorSearchStep from index steps module - Remove unused prepare_start_config and should_precheck_start exports - Move import statements to proper location in reme.py - Update test module to use direct import path for CliService - Remove vector_search and bm25_search configurations from jinli_lme.yaml - Add workspace directory environment variable configuration - Add docstring to getcwd method in agent wrapper - Remove empty middleware list from agent wrapper initialization * feat(index): add BM25 and vector search steps with tool context deduplication - Add Bm25SearchStep for plain BM25 keyword search with tool_context deduplication - Add VectorSearchStep for plain vector search with tool_context deduplication - Implement tool context state management with TTL-based deduplication - Add support for chunk deduplication across tool contexts within TTL window - Update index steps module to include new search step classes - Add test coverage for CLI metadata output functionality - Refactor CLI service to remove unused show_status parameter - Update documentation comments to reflect internal service configuration * feat(steps): add Python code execution capability - Introduce PythonExecuteStep to run Python code in subprocess - Add configuration for python_execute step in jinli_lme.yaml - Register python_execute in available tools list - Implement timeout handling with default 60 second limit - Capture stdout/stderr output and return code metadata - Add comprehensive unit tests for execution scenarios - Support workspace directory context for code execution - Handle timeout errors and runtime exceptions gracefully * refactor(python_execute): replace subprocess with asyncio for Python code execution - Replace subprocess.run with asyncio.create_subprocess_exec for non-blocking execution - Add _PythonResult dataclass to encapsulate execution results and timeout status - Implement proper timeout handling with asyncio.wait_for and process.kill() - Update metadata to include returncode and stderr when timeout occurs - Convert synchronous _run_python method to asynchronous implementation - Maintain backward compatibility while improving execution reliability * refactor(python_execute): replace subprocess with asyncio for Python code execution - Replace subprocess.run with asyncio.create_subprocess_exec for non-blocking execution - Add _PythonResult dataclass to encapsulate execution results and timeout status - Implement proper timeout handling with asyncio.wait_for and process.kill() - Update metadata to include returncode and stderr when timeout occurs - Convert synchronous _run_python method to asynchronous implementation - Maintain backward compatibility while improving execution reliability
117 lines
4.1 KiB
Python
117 lines
4.1 KiB
Python
"""CLI service: run one configured job locally, then exit."""
|
|
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from .base_service import BaseService
|
|
from ..component_registry import R
|
|
from ..job import BaseJob
|
|
from ...config import resolve_app_config
|
|
from ...schema import ApplicationConfig
|
|
from ...utils import get_logger
|
|
|
|
if TYPE_CHECKING:
|
|
from ...application import Application
|
|
|
|
_APP_CONFIG_KEYS = set(ApplicationConfig.model_fields)
|
|
|
|
|
|
def prepare_start_config(kwargs: dict) -> dict:
|
|
"""Resolve ``reme start`` kwargs, translating top-level ``job=...`` into internal cli service config."""
|
|
if "job" not in kwargs:
|
|
return resolve_app_config(**kwargs)
|
|
return _prepare_job_start_config(dict(kwargs))
|
|
|
|
|
|
def should_precheck_start(config: dict) -> bool:
|
|
"""CLI service does not bind a port, so it should skip service port prechecks."""
|
|
service = config.get("service")
|
|
return not (isinstance(service, dict) and service.get("backend") == "cli")
|
|
|
|
|
|
def _prepare_job_start_config(kwargs: dict) -> dict:
|
|
"""Translate ``reme start job=...`` args into the internal cli service fields."""
|
|
job = kwargs.pop("job")
|
|
config_kwargs: dict = {}
|
|
job_args: dict = {}
|
|
|
|
for key, value in kwargs.items():
|
|
if key == "config" or key in _APP_CONFIG_KEYS:
|
|
config_kwargs[key] = value
|
|
else:
|
|
job_args[key] = value
|
|
|
|
# One-shot CLI jobs should print only their answer by default. Reconfigure
|
|
# before resolve_app_config() so even config-loading logs stay off stdout.
|
|
if "log_to_console" not in config_kwargs:
|
|
get_logger(log_to_console=False, log_to_file=False, force_init=True)
|
|
|
|
config = resolve_app_config(**config_kwargs)
|
|
if "enable_logo" not in config_kwargs:
|
|
config["enable_logo"] = False
|
|
if "log_to_console" not in config_kwargs:
|
|
config["log_to_console"] = False
|
|
service = dict(config.get("service") or {})
|
|
service.update({"backend": "cli", "job": job, "job_args": job_args})
|
|
config["service"] = service
|
|
return config
|
|
|
|
|
|
@R.register("cli")
|
|
class CliService(BaseService):
|
|
"""Execute a single job through the normal application lifecycle without serving a port."""
|
|
|
|
def __init__(
|
|
self,
|
|
job: str = "",
|
|
job_args: dict[str, Any] | None = None,
|
|
show_metadata: bool = False,
|
|
**kwargs,
|
|
):
|
|
super().__init__(**kwargs)
|
|
self.job = job
|
|
self.job_args = job_args or {}
|
|
self.show_metadata = show_metadata
|
|
|
|
def build_service(self, app: "Application") -> None:
|
|
"""No network framework is needed for local CLI execution."""
|
|
self.service = None
|
|
|
|
def add_job(self, job: BaseJob) -> bool:
|
|
"""CLI execution does not register jobs; Application already owns them."""
|
|
return False
|
|
|
|
def start_service(self, app: "Application") -> None:
|
|
"""Run the configured job once and print the same human-facing answer style as CLI clients."""
|
|
asyncio.run(self._run_job(app))
|
|
|
|
def run_app(self, app: "Application") -> None:
|
|
"""Bypass BaseService.add_jobs(), which is only meaningful for serving protocols."""
|
|
self.build_service(app)
|
|
self.start_service(app)
|
|
|
|
async def _run_job(self, app: "Application") -> None:
|
|
if not self.job:
|
|
raise ValueError("cli service requires service.job")
|
|
|
|
await app.start()
|
|
try:
|
|
response = await app.run_job(self.job, **self.job_args)
|
|
output = self._format_response(response.answer, response.metadata)
|
|
if response.success:
|
|
print(output)
|
|
else:
|
|
print(output, file=sys.stderr)
|
|
raise SystemExit(1)
|
|
finally:
|
|
await app.close()
|
|
|
|
def _format_response(self, answer: Any, metadata: dict | None) -> str:
|
|
if not isinstance(answer, str):
|
|
answer = json.dumps(answer, ensure_ascii=False, indent=2)
|
|
parts = [answer]
|
|
if self.show_metadata and metadata:
|
|
parts.append(json.dumps(metadata, ensure_ascii=False))
|
|
return "\n".join(parts)
|