diff --git a/.github/workflows/ci-python-quality.yml b/.github/workflows/ci-python-quality.yml index 21cd7a87..d39302f6 100644 --- a/.github/workflows/ci-python-quality.yml +++ b/.github/workflows/ci-python-quality.yml @@ -34,7 +34,7 @@ jobs: - name: Install run: | pip install -q -e reme_studio -e ".[dev,core]" - pip install -q --no-deps -e plugins/auto-fin -e plugins/daily_paper + pip install -q --no-deps -e plugins/auto-fin -e plugins/daily_paper -e plugins/lme -e plugins/beam - name: Pre-commit starts run: pre-commit run --all-files diff --git a/.github/workflows/ci-python-tests.yml b/.github/workflows/ci-python-tests.yml index 00b3bbbf..9aab1e75 100644 --- a/.github/workflows/ci-python-tests.yml +++ b/.github/workflows/ci-python-tests.yml @@ -40,11 +40,12 @@ jobs: pip install -e reme_studio -e ".[dev,core]" pip install --no-deps -e plugins/auto-fin pip install -e plugins/daily_paper + pip install -e plugins/lme -e plugins/beam pip install coverage - name: Run unit tests run: | - coverage run -m pytest tests/unit plugins/auto-fin plugins/daily_paper \ + coverage run -m pytest tests/unit plugins/auto-fin plugins/daily_paper plugins/lme plugins/beam \ -v \ --tb=long \ -s \ diff --git a/AGENTS.md b/AGENTS.md index 169e2b4f..7718891a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,8 +46,8 @@ and concise documentation together. - `reme/components/service/`: local CLI, HTTP, and MCP service backends. - `reme/components/`: agent wrappers, model adapters, stores, catalogs, graphs, indexes, clients, tokenizers, and outbound proxies. -- `reme/steps/`: registered job steps grouped by common, file I/O, index, evolve, cookbook, benchmark, and transfer - concerns. +- `reme/steps/`: registered job steps grouped by common, file I/O, index, evolve, cookbook, and transfer + concerns, plus shared benchmark base classes under `benchmark/`. - `reme/utils/`: shared utilities, including service discovery, logging, web-static resolution, session I/O, token accounting, and wikilink handling. - `tests/unit/`: primary fast, isolated validation suite. @@ -56,7 +56,7 @@ and concise documentation together. `@agentscope-ai/reme_studio` npm static distribution. - `typescript/`: the independently published `@agentscope-ai/reme` package, including the shared TypeScript client and DeepSeek Harness and OpenClaw adapters. -- `plugins/`: installable ReMe extensions, such as Auto Fin. +- `plugins/`: installable ReMe extensions, including Auto Fin and LME/BEAM benchmark Steps and application presets. - `integrations/`: adapters that connect ReMe to external agent hosts, such as Claude Code, DSH, and Hermes Agent. - `skills/`: standalone skills; `reme_memory` calls ReMe, while other skills may use separate tools or direct-file conventions. diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index aef8a3b7..014427c2 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -15,8 +15,20 @@ include abstention, contradiction resolution, event ordering, information extraction, instruction following, knowledge update, multi-session reasoning, preference following, summarization, and temporal reasoning. -> For the shared setup (dependencies, credentials, log conventions) see the -> [top-level benchmark README](../README.md). +Install ReMe and the BEAM plugin in editable mode from the repository root: + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/beam --editable +reme plugins validate beam +``` + +The runner explicitly enables the installed `beam` plugin and combines its defaults with +ReMe's built-in `benchmark` preset. Editable installation keeps changes under +[`plugins/beam`](../../plugins/beam/README.md) visible without reinstalling the plugin. +Custom application config paths still work through `reme.config` and can use `extends: benchmark`. +This directory continues to own the runner, evaluation settings, dataset and outputs. +Model credentials use the environment variables declared by the shared benchmark configuration. ## 1. Get the Dataset @@ -59,7 +71,7 @@ python benchmark/beam/run.py --eval_only # reuse existing workspac | `dataset.start_index` / `num_items` | Case pagination (`num_items` `0` = all). | | `dataset.workspace_root` | Per-case workspace root (`benchmark/beam/workspaces/beam`). | | `evaluation.num_workers` | `0` = auto, `1` = sequential, `>1` = parallel. | -| `reme.config` | ReMe config used (`beam.yaml`). | +| `reme.config` | ReMe config used (`benchmark`). | | `output.dir` | Results directory (`benchmark/beam/results`). | ## 5. Outputs diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index f562193b..1c063497 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -13,7 +13,19 @@ ordering（事件排序）、information extraction（信息抽取）、instruct knowledge update（知识更新）、multi-session reasoning（多会话推理）、preference following （偏好遵循）、summarization（摘要）与 temporal reasoning（时间推理）。 -> 公共设置（依赖、凭据、日志约定）见[总评测说明](../README_ZH.md)。 +在仓库根目录以 editable 模式安装 ReMe 和 BEAM 插件： + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/beam --editable +reme plugins validate beam +``` + +runner 显式启用已安装的 `beam` 插件，并将插件默认配置与 ReMe 内置的 `benchmark` 配置组合。 +editable 安装会让 [`plugins/beam`](../../plugins/beam/README_ZH.md) 下的源码修改直接生效，无需重复安装。 +本目录继续保留评测参数、数据集及输出。自定义完整应用配置路径仍可通过 `reme.config` 指定， +并可使用 `extends: benchmark`。 +模型凭据通过公共 benchmark 配置中声明的环境变量设置。 ## 1. 获取数据集 @@ -55,7 +67,7 @@ python benchmark/beam/run.py --eval_only # 复用已有工作区 | `dataset.start_index` / `num_items` | case 分页（`num_items` 为 `0` 表示全部）。 | | `dataset.workspace_root` | case 工作区根目录（`benchmark/beam/workspaces/beam`）。 | | `evaluation.num_workers` | `0` = 自动，`1` = 串行，`>1` = 并行。 | -| `reme.config` | 使用的 ReMe 配置（`beam.yaml`）。 | +| `reme.config` | 使用的 ReMe 配置（`benchmark`）。 | | `output.dir` | 结果目录（`benchmark/beam/results`）。 | ## 5. 输出 diff --git a/benchmark/beam/config.yaml b/benchmark/beam/config.yaml index 423697a5..cca633b8 100644 --- a/benchmark/beam/config.yaml +++ b/benchmark/beam/config.yaml @@ -14,7 +14,7 @@ evaluation: compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression reme: - config: "beam.yaml" # reme config (in reme/config/) + config: "benchmark" # shared ReMe benchmark preset; runner enables the installed beam plugin output: dir: "benchmark/beam/results" diff --git a/benchmark/beam/run.py b/benchmark/beam/run.py index 06e35b7d..b49284fd 100644 --- a/benchmark/beam/run.py +++ b/benchmark/beam/run.py @@ -29,7 +29,7 @@ import yaml from dotenv import load_dotenv # Load .env from project root -_PROJECT_ROOT = Path(__file__).parent.parent.parent +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent load_dotenv(_PROJECT_ROOT / ".env") # Workspace root — read from config.yaml (dataset.workspace_root) @@ -151,6 +151,22 @@ def load_eval_config(config_path: str | None = None) -> dict: return yaml.safe_load(raw) +def create_reme_app(config: str = "benchmark", **overrides): + """Create an app with the installed BEAM plugin explicitly enabled. + + Plugin discovery remains environment-based; editable installation keeps local + plugin source changes visible to every multiprocessing worker. + """ + from reme import Application + from reme.config import resolve_app_config + + enabled_plugins = list(overrides.pop("plugins", ()) or ()) + if "beam" not in enabled_plugins: + enabled_plugins.append("beam") + app_config = resolve_app_config(config=config, plugins=enabled_plugins, **overrides) + return Application(**app_config) + + # --------------------------------------------------------------------------- # BEAM data loading # --------------------------------------------------------------------------- @@ -319,8 +335,6 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False Returns: A results dict with all questions, answers, and judgments. """ - from reme import Application - from reme.config import resolve_app_config dataset_cfg = eval_config["dataset"] chat_size = dataset_cfg["chat_size"] @@ -375,7 +389,7 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False force_init=True, ) - cfg = resolve_app_config( + app = create_reme_app( config=eval_config["reme"]["config"], workspace_dir=workspace_dir, log_to_console=output_cfg.get("log_to_console", True), @@ -383,7 +397,6 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False enable_logo=False, ) - app = Application(**cfg) await app.start() from reme.utils.evaluation_interface import check_agent_token_usage # noqa: E402 diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index 070f921e..7f07d170 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -12,8 +12,20 @@ agentic (ReAct) mode, and scores the answer with an LLM-as-judge. Question types include single-session (user / assistant / preference), multi-session reasoning, knowledge update, and temporal reasoning. -> For the shared setup (dependencies, credentials, log conventions) see the -> [top-level benchmark README](../README.md). +Install ReMe and the LongMemEval plugin in editable mode from the repository root: + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/lme --editable +reme plugins validate lme +``` + +The runner explicitly enables the installed `lme` plugin and combines its defaults with +ReMe's built-in `benchmark` preset. Editable installation keeps changes under +[`plugins/lme`](../../plugins/lme/README.md) visible without reinstalling the plugin. +Custom application config paths still work through `reme.config` and can use `extends: benchmark`. +This directory continues to own the runner, evaluation settings, dataset and outputs. +Model credentials use the environment variables declared by the shared benchmark configuration. ## 1. Get the Dataset @@ -46,7 +58,8 @@ python benchmark/longmemeval/run.py --eval_only # reuse existing w 1. Load the dataset (ground truth is embedded in the data file). 2. For each item, create an isolated workspace and ingest sessions in chronological order. -3. Trigger `auto_dream` when consecutive sessions cross the configured hour (default 23:00). +3. If a custom application configuration enables `auto_dream`, trigger it when sessions cross the configured hour + (default 23:00). The packaged preset leaves it disabled. 4. Answer each question via agentic (ReAct) mode. 5. Judge the answer (binary yes/no) with the `answer_judge` job and print per-type accuracy. @@ -60,7 +73,7 @@ python benchmark/longmemeval/run.py --eval_only # reuse existing w | `dataset.workspace_root` | Per-item workspace root (`benchmark/longmemeval/workspaces/longmemeval-s`). | | `evaluation.num_workers` | `0` = auto (cpu-2), `1` = sequential, `>1` = parallel. | | `evaluation.filter_future_sessions` | Only ingest sessions with timestamp ≤ `question_date`. | -| `reme.config` | ReMe config used (`lme.yaml`). | +| `reme.config` | ReMe config used (`benchmark`). | | `reme.dream_trigger_hour` / `dream_scan_days` / `dream_max_units` | Dream triggering behavior. | | `output.dir` | Results directory (`benchmark/longmemeval/results`). | @@ -93,4 +106,4 @@ agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only (reusing prebuilt | single-session-preference | 0.633 | 36,802 | 818 | 37,620 | 3.60 | | single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 | | temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 | -| **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** | \ No newline at end of file +| **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** | diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index 9c07b175..4c66df67 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -8,7 +8,19 @@ LongMemEval 是一个面向**多轮多会话历史的长期记忆能力**的评 题型包括单会话（user / assistant / preference）、多会话推理、知识更新与时间推理等。 -> 公共设置（依赖、凭据、日志约定）见[总评测说明](../README_ZH.md)。 +在仓库根目录以 editable 模式安装 ReMe 和 LongMemEval 插件： + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/lme --editable +reme plugins validate lme +``` + +runner 显式启用已安装的 `lme` 插件，并将插件默认配置与 ReMe 内置的 `benchmark` 配置组合。 +editable 安装会让 [`plugins/lme`](../../plugins/lme/README_ZH.md) 下的源码修改直接生效，无需重复安装。 +本目录继续保留评测参数、数据集及输出。自定义完整应用配置路径仍可通过 `reme.config` 指定， +并可使用 `extends: benchmark`。 +模型凭据通过公共 benchmark 配置中声明的环境变量设置。 ## 1. 获取数据集 @@ -41,7 +53,7 @@ python benchmark/longmemeval/run.py --eval_only # 复用已有工 1. 加载数据集（ground truth 已内嵌在数据文件中）。 2. 为每个条目创建独立工作区，按时间顺序摄入会话。 -3. 当相邻会话跨越配置的时刻（默认 23:00）时触发 `auto_dream`。 +3. 若自定义应用配置启用了 `auto_dream`，在相邻会话跨越配置时刻（默认 23:00）时触发；插件预设保持关闭。 4. 以 agentic（ReAct）模式回答每个问题。 5. 通过 `answer_judge` 任务对答案做二元（yes/no）评判，并输出各类型准确率。 @@ -55,7 +67,7 @@ python benchmark/longmemeval/run.py --eval_only # 复用已有工 | `dataset.workspace_root` | 条目工作区根目录（`benchmark/longmemeval/workspaces/longmemeval-s`）。 | | `evaluation.num_workers` | `0` = 自动（cpu-2），`1` = 串行，`>1` = 并行。 | | `evaluation.filter_future_sessions` | 仅摄入时间戳 ≤ `question_date` 的会话。 | -| `reme.config` | 使用的 ReMe 配置（`lme.yaml`）。 | +| `reme.config` | 使用的 ReMe 配置（`benchmark`）。 | | `reme.dream_trigger_hour` / `dream_scan_days` / `dream_max_units` | dream 触发行为。 | | `output.dir` | 结果目录（`benchmark/longmemeval/results`）。 | diff --git a/benchmark/longmemeval/config.yaml b/benchmark/longmemeval/config.yaml index 72ee7b75..36260479 100644 --- a/benchmark/longmemeval/config.yaml +++ b/benchmark/longmemeval/config.yaml @@ -10,7 +10,7 @@ dataset: workspace_root: "benchmark/longmemeval/workspaces/longmemeval-s" # workspace root for item workspaces evaluation: - # LLM-as-judge uses the 'judge' as_llm component defined in lme.yaml + # LLM-as-judge uses the 'judge' as_llm component defined in benchmark.yaml # Model and credentials are configured there (reading from .env) # Judgment is always binary (yes/no) — defined in lme/llm_judge.yaml num_workers: 32 # 0 = auto (cpu_count - 2, min 1); 1 = sequential; >1 = parallel @@ -18,7 +18,7 @@ evaluation: compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression reme: - config: "lme.yaml" # reme config to use (in reme/config/) + config: "benchmark" # shared ReMe benchmark preset; runner enables the installed lme plugin # Dream trigger: when gap between consecutive sessions crosses this hour (23:00) dream_trigger_hour: 23 # Dream scan_days for each trigger diff --git a/benchmark/longmemeval/run.py b/benchmark/longmemeval/run.py index 9e48af43..97b10ae4 100644 --- a/benchmark/longmemeval/run.py +++ b/benchmark/longmemeval/run.py @@ -28,7 +28,7 @@ import yaml from dotenv import load_dotenv # Load .env from project root -_PROJECT_ROOT = Path(__file__).parent.parent.parent +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent load_dotenv(_PROJECT_ROOT / ".env") # Workspace root for evaluation items — read from config.yaml (dataset.workspace_root) @@ -150,6 +150,22 @@ def load_eval_config(config_path: str | None = None) -> dict: return yaml.safe_load(raw) +def create_reme_app(config: str = "benchmark", **overrides): + """Create an app with the installed LongMemEval plugin explicitly enabled. + + Plugin discovery remains environment-based; editable installation keeps local + plugin source changes visible to every multiprocessing worker. + """ + from reme import Application + from reme.config import resolve_app_config + + enabled_plugins = list(overrides.pop("plugins", ()) or ()) + if "lme" not in enabled_plugins: + enabled_plugins.append("lme") + app_config = resolve_app_config(config=config, plugins=enabled_plugins, **overrides) + return Application(**app_config) + + # --------------------------------------------------------------------------- # Date utilities # --------------------------------------------------------------------------- @@ -257,8 +273,6 @@ async def evaluate_item(item: dict, eval_config: dict, item_index: int, eval_onl using the existing workspace. Useful for re-evaluating different query configurations without re-ingesting sessions. """ - from reme import Application - from reme.config import resolve_app_config from reme.utils.evaluation_interface import track_agent_token_usage, track_job_counts reme_cfg = eval_config["reme"] @@ -325,7 +339,7 @@ async def evaluate_item(item: dict, eval_config: dict, item_index: int, eval_onl force_init=True, ) - cfg = resolve_app_config( + app = create_reme_app( config=reme_cfg["config"], workspace_dir=workspace_dir, log_to_console=output_cfg.get("log_to_console", True), @@ -333,7 +347,6 @@ async def evaluate_item(item: dict, eval_config: dict, item_index: int, eval_onl enable_logo=False, ) - app = Application(**cfg) await app.start() try: diff --git a/benchmark/pibench/README.md b/benchmark/pibench/README.md index c62249a0..858e689c 100644 --- a/benchmark/pibench/README.md +++ b/benchmark/pibench/README.md @@ -289,7 +289,7 @@ removed"). They need the executed tool calls in the trace. The pipeline: | user_agent / judger models | `config/models/reme.yaml` | | Agent system prompt | `bridge_reme.py` `build_system_prompt()` | | Memory retrieval limit/threshold | `--search-limit/--search-min-score` on the bridge command in `run_persona.sh` | -| ReMe internal parameters | **Do not modify ReMe source**; write a dedicated config modeled on `reme/config/beam.yaml` and override via `resolve_app_config(config=...)` (see bridge `_init_reme_app`) | +| ReMe internal parameters | **Do not modify ReMe source**; extend the built-in `benchmark` config and override via `resolve_app_config(config=...)` (see bridge `_init_reme_app`) | | Turn timeout / tool iteration cap | `config/models/reme.yaml` `run.turn_timeout`, `model.max_tool_iterations` | ## 11. Troubleshooting diff --git a/benchmark/pibench/README_ZH.md b/benchmark/pibench/README_ZH.md index 0a8b58d6..5e4668b2 100644 --- a/benchmark/pibench/README_ZH.md +++ b/benchmark/pibench/README_ZH.md @@ -255,7 +255,7 @@ grep -h "overall_average_score\|overall_proactiveness" \ | user_agent / judger 模型 | `config/models/reme.yaml` | | agent system prompt | `bridge_reme.py` `build_system_prompt()` | | 记忆检索条数/阈值 | `run_persona.sh` bridge 启动命令的 `--search-limit/--search-min-score` | -| ReMe 内部参数 | **不要改 ReMe 源码**；仿照 `reme/config/beam.yaml` 写专有配置，经 `resolve_app_config(config=...)` 覆盖（见 bridge `_init_reme_app`） | +| ReMe 内部参数 | **不要改 ReMe 源码**；继承内置 `benchmark` 配置，并经 `resolve_app_config(config=...)` 覆盖（见 bridge `_init_reme_app`） | | 轮超时/工具迭代上限 | `config/models/reme.yaml` `run.turn_timeout`、`model.max_tool_iterations` | ## 11. 故障排查 diff --git a/docs/en/plugin_management.md b/docs/en/plugin_management.md index 31b8400a..a2429787 100644 --- a/docs/en/plugin_management.md +++ b/docs/en/plugin_management.md @@ -176,6 +176,16 @@ When the application uses an MCP service, service-enabled plugin Jobs appear as Custom application configs must provide the plugin's runtime dependencies, including an `agent_wrapper.default` and the `search` and `read` Jobs used by Auto Fin. +## Benchmark application presets + +The [LME](../../plugins/lme/README.md) and [BEAM](../../plugins/beam/README.md) plugins +register their backends and plugin-owned Jobs in `plugin.yaml`. ReMe's built-in `benchmark` +preset provides the shared core Jobs and components without inheriting `default`, so default +background and cron jobs are not included. Install the selected benchmark plugin, then use +`config=benchmark` together with `plugins=["lme"]` or `plugins=["beam"]`. The repository's +benchmark runners enable the corresponding installed plugin automatically; editable installation +keeps local plugin changes visible. Dataset runners remain under `benchmark/`. + ## Uninstall a plugin Use the plugin entry-point name, not necessarily the distribution name: diff --git a/docs/zh/plugin_management.md b/docs/zh/plugin_management.md index f9f735f8..a0cdfde7 100644 --- a/docs/zh/plugin_management.md +++ b/docs/zh/plugin_management.md @@ -169,6 +169,15 @@ curl -s http://127.0.0.1:2333/auto_fin \ 自定义应用配置需要提供插件的运行依赖，包括 `agent_wrapper.default`，以及 Auto Fin 使用的 `search` 和 `read` Jobs。 +## Benchmark 应用配置 + +[LME](../../plugins/lme/README_ZH.md) 和 [BEAM](../../plugins/beam/README_ZH.md) 插件通过 +`plugin.yaml` 注册 backend 和插件拥有的 Job。ReMe 内置的 `benchmark` 配置提供公共核心 Job 和 +Component，并且不继承 `default`，因此不包含默认后台和定时任务。先安装所需的 benchmark 插件， +再使用 `config=benchmark`，同时指定 `plugins=["lme"]` 或 `plugins=["beam"]`。仓库内的 benchmark +runner 会自动启用对应的已安装插件；editable 安装可让本地源码修改直接生效。数据集 runner 仍位于 +`benchmark/`。 + ## 卸载插件 这里使用插件 entry-point 名称，它不一定等于 distribution 名称： diff --git a/plugins/beam/LICENSE b/plugins/beam/LICENSE new file mode 100644 index 00000000..65c2c5cf --- /dev/null +++ b/plugins/beam/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Alibaba Group + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/beam/README.md b/plugins/beam/README.md new file mode 100644 index 00000000..fb2f9559 --- /dev/null +++ b/plugins/beam/README.md @@ -0,0 +1,38 @@ +# BEAM plugin + +[中文说明](./README_ZH.md) + +This plugin owns the BEAM memory, agentic-answer and judge Steps, their prompts, +and their Job defaults in `plugin.yaml`. ReMe's built-in `benchmark.yaml` owns the +shared evaluation Jobs and components. Dataset handling, the runner and results +remain in [`benchmark/beam`](../../benchmark/beam/README.md). + +From the repository root, install ReMe and this plugin in editable mode before running the benchmark: + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/beam --editable +reme plugins validate beam +python benchmark/beam/run.py +``` + +Editable installation registers the `beam` entry point while keeping source changes immediately +visible. The runner selects the built-in `benchmark` preset and explicitly enables `beam` for +each Application. Installing the plugin makes it discoverable but does not enable it globally. + +`plugin.yaml` registers backends and contributes the plugin-owned `auto_memory`, +`agentic_answer` and `answer_judge` Job defaults. Start the installed plugin with +`reme start config=benchmark plugins='["beam"]'`. The shared preset does not inherit +`default`: only declared Jobs run, indexing is manual, and neither scheduled dream +nor the optional `auto_dream` Job is enabled. +The existing `auto_memory`, `agentic_answer`, `answer_judge`, `bench` and `judge` +names and model environment variables are unchanged. Explicit application/CLI overrides +still take precedence. Installing this plugin does not start an evaluation. + +The shared answer base class lives in `reme.steps.benchmark.base_agentic_answer`. +The old core-owned `reme.steps.benchmark.beam` Python import path is removed. +Custom Python callers should import memory, search and answer Steps from `reme_beam`, and the +judge Step from `judge_beam`. After uninstalling, +Applications and CLI services must omit the plugin until it is installed again. +Uninstallation never removes datasets, workspaces or results. +Restart an existing service after changing plugins. diff --git a/plugins/beam/README_ZH.md b/plugins/beam/README_ZH.md new file mode 100644 index 00000000..4d695715 --- /dev/null +++ b/plugins/beam/README_ZH.md @@ -0,0 +1,32 @@ +# BEAM 插件 + +[English](./README.md) + +插件包含 BEAM 的记忆、回答、评分 Step、提示词，以及 `plugin.yaml` 中对应的 Job 默认配置。 +ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集处理、runner 和结果仍留在 +[`benchmark/beam`](../../benchmark/beam/README_ZH.md)。 + +在仓库根目录以 editable 模式安装 ReMe 和本插件，再运行评测： + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/beam --editable +reme plugins validate beam +python benchmark/beam/run.py +``` + +editable 安装会注册 `beam` entry point，并让源码修改立即生效。runner 选择内置 `benchmark` +配置，并为每个 Application 显式启用 `beam`。安装只让插件可被发现，不会在所有应用中全局启用。 + +`plugin.yaml` 注册 backend，并通过 `application_defaults` 提供插件拥有的 `auto_memory`、 +`agentic_answer` 和 `answer_judge` Job。安装后使用 +`reme start config=benchmark plugins='["beam"]'`。公共评测配置不继承 `default`，只运行声明的 Job： +索引手动更新，dream 定时任务和可选的 `auto_dream` +均保持关闭。原有 `auto_memory`、`agentic_answer`、`answer_judge`、`bench`、`judge` 名称及模型环境变量 +保持不变，显式应用参数和 CLI 覆盖仍优先。安装或启用插件不会自动开始评测。 + +共享回答基类位于 `reme.steps.benchmark.base_agentic_answer`。 +原 `reme.steps.benchmark.beam` Python 导入路径已移除。自定义 Python 调用应从 `reme_beam` +导入记忆、搜索和回答 Step，并从 `judge_beam` 导入评判 Step。 +卸载插件后，Application 和 CLI 服务必须移除插件选择，直到再次安装。 +卸载不会删除数据集、工作区或结果。修改插件后需重启已有服务。 diff --git a/plugins/beam/pyproject.toml b/plugins/beam/pyproject.toml new file mode 100644 index 00000000..2b35f5aa --- /dev/null +++ b/plugins/beam/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "reme-beam" +version = "0.1.0" +description = "BEAM benchmark plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.11" +dependencies = [ + "reme-ai[as]>=0.4.1.11", + "json-repair", + "numpy>=2.2.6", +] + +[project.entry-points."reme.plugins"] +beam = "reme_beam" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["reme_beam*", "judge_beam*"] + +[tool.setuptools.package-data] +reme_beam = ["plugin.yaml", "*.yaml"] +judge_beam = ["*.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/beam/src/judge_beam/__init__.py b/plugins/beam/src/judge_beam/__init__.py new file mode 100644 index 00000000..7090e1ce --- /dev/null +++ b/plugins/beam/src/judge_beam/__init__.py @@ -0,0 +1,5 @@ +"""BEAM benchmark judge backend.""" + +from .llm_judge import BeamRubricJudgeStep + +__all__ = ["BeamRubricJudgeStep"] diff --git a/reme/steps/benchmark/beam/llm_judge.py b/plugins/beam/src/judge_beam/llm_judge.py similarity index 98% rename from reme/steps/benchmark/beam/llm_judge.py rename to plugins/beam/src/judge_beam/llm_judge.py index 879c82e3..fdadabbb 100644 --- a/reme/steps/benchmark/beam/llm_judge.py +++ b/plugins/beam/src/judge_beam/llm_judge.py @@ -24,10 +24,9 @@ from typing import List, Tuple import numpy as np from json_repair import repair_json -from ...base_step import BaseStep, Ref -from ....components import R -from ....components.as_embedding import BaseAsEmbedding -from ....enumeration import ComponentEnum +from reme.steps.base_step import BaseStep, Ref +from reme.components.as_embedding import BaseAsEmbedding +from reme.enumeration import ComponentEnum # --------------------------------------------------------------------------- @@ -231,7 +230,6 @@ def _event_ordering_score( } -@R.register("beam_rubric_judge_step") class BeamRubricJudgeStep(BaseStep): """Judge an LLM response against a list of rubric criteria. diff --git a/reme/steps/benchmark/beam/llm_judge.yaml b/plugins/beam/src/judge_beam/llm_judge.yaml similarity index 100% rename from reme/steps/benchmark/beam/llm_judge.yaml rename to plugins/beam/src/judge_beam/llm_judge.yaml diff --git a/reme/steps/benchmark/beam/__init__.py b/plugins/beam/src/reme_beam/__init__.py similarity index 56% rename from reme/steps/benchmark/beam/__init__.py rename to plugins/beam/src/reme_beam/__init__.py index e8c6e48f..2a5e8a7d 100644 --- a/reme/steps/benchmark/beam/__init__.py +++ b/plugins/beam/src/reme_beam/__init__.py @@ -1,11 +1,11 @@ -"""BEAM benchmark steps.""" +"""BEAM benchmark backends and application configuration for ReMe.""" from .agentic_answer import BeamAgenticAnswerStep -from .llm_judge import BeamRubricJudgeStep from .auto_memory import BeamAutoMemoryStep +from .search_v2 import SearchV2Step __all__ = [ "BeamAgenticAnswerStep", - "BeamRubricJudgeStep", "BeamAutoMemoryStep", + "SearchV2Step", ] diff --git a/reme/steps/benchmark/beam/agentic_answer.py b/plugins/beam/src/reme_beam/agentic_answer.py similarity index 80% rename from reme/steps/benchmark/beam/agentic_answer.py rename to plugins/beam/src/reme_beam/agentic_answer.py index 3b9a02fc..d6452ec0 100644 --- a/reme/steps/benchmark/beam/agentic_answer.py +++ b/plugins/beam/src/reme_beam/agentic_answer.py @@ -1,10 +1,8 @@ """BEAM agentic answer step – ReAct agent that answers questions using the search tool.""" -from ....components import R -from ..base import BaseAgenticAnswerStep +from reme.steps.benchmark import BaseAgenticAnswerStep -@R.register("beam_agentic_answer_step") class BeamAgenticAnswerStep(BaseAgenticAnswerStep): """Answer a BEAM probing question via ReAct agent with access to the search tool. diff --git a/reme/steps/benchmark/beam/agentic_answer.yaml b/plugins/beam/src/reme_beam/agentic_answer.yaml similarity index 100% rename from reme/steps/benchmark/beam/agentic_answer.yaml rename to plugins/beam/src/reme_beam/agentic_answer.yaml diff --git a/reme/steps/benchmark/beam/auto_memory.py b/plugins/beam/src/reme_beam/auto_memory.py similarity index 98% rename from reme/steps/benchmark/beam/auto_memory.py rename to plugins/beam/src/reme_beam/auto_memory.py index cccb0c28..378f97bf 100644 --- a/reme/steps/benchmark/beam/auto_memory.py +++ b/plugins/beam/src/reme_beam/auto_memory.py @@ -4,9 +4,8 @@ from datetime import datetime, timedelta from agentscope.message import Msg -from ...evolve.auto_memory import AutoMemoryStep, _normalize_msg_timestamp -from ...file_io import validate_session_id -from ....components import R +from reme.steps.evolve.auto_memory import AutoMemoryStep, _normalize_msg_timestamp +from reme.steps.file_io import validate_session_id # Runtime-context key carrying the 0-based line offset of the current segment # inside the full session file (segmented ingestion of long sessions). @@ -168,7 +167,6 @@ def _interpolate_timestamps(items: list[dict]) -> list[dict]: return result -@R.register("beam_auto_memory_step") class BeamAutoMemoryStep(AutoMemoryStep): """AutoMemoryStep variant that interpolates timestamps for BEAM sessions. diff --git a/reme/steps/benchmark/beam/auto_memory.yaml b/plugins/beam/src/reme_beam/auto_memory.yaml similarity index 100% rename from reme/steps/benchmark/beam/auto_memory.yaml rename to plugins/beam/src/reme_beam/auto_memory.yaml diff --git a/plugins/beam/src/reme_beam/plugin.yaml b/plugins/beam/src/reme_beam/plugin.yaml new file mode 100644 index 00000000..389653fc --- /dev/null +++ b/plugins/beam/src/reme_beam/plugin.yaml @@ -0,0 +1,120 @@ +backends: + beam_auto_memory_step: reme_beam.auto_memory:BeamAutoMemoryStep + beam_agentic_answer_step: reme_beam.agentic_answer:BeamAgenticAnswerStep + beam_rubric_judge_step: judge_beam.llm_judge:BeamRubricJudgeStep + beam_search_v2_step: reme_beam.search_v2:SearchV2Step + +application_defaults: + jobs: + search: + backend: base + description: "Hybrid workspace search (vector + BM25, RRF-fused) with deduplication." + parameters: + type: object + properties: + query: + type: string + description: "search query" + start_date: + type: string + description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded" + end_date: + type: string + description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded" + required: + - query + steps: + - backend: beam_search_v2_step + vector_weight: 0.7 + candidate_multiplier: 5.0 + expand_links: false + max_links_per_direction: 10 + + agentic_answer: + backend: base + description: "BEAM agentic answer job (ReAct agent with search tool)" + watch_dirs: [] + watch_suffixes: [] + parameters: + type: object + properties: + query: + type: string + description: "The query to ask" + query_time: + type: string + description: "ISO timestamp representing the query time" + default: "" + required: + - query + steps: + - backend: beam_agentic_answer_step + agent_wrapper: bench + + answer_judge: + backend: base + description: "BEAM rubric-based LLM-as-Judge: evaluate response against rubric criteria" + watch_dirs: [] + watch_suffixes: [] + parameters: + type: object + properties: + llm_response: + type: string + description: "The model's response to evaluate" + rubric: + type: array + description: "List of rubric criteria to check" + items: + type: string + probing_question: + type: string + description: "The original probing question" + default: "" + question_type: + type: string + description: "BEAM question type (e.g. event_ordering)" + default: "" + required: + - llm_response + - rubric + steps: + - backend: beam_rubric_judge_step + agent_wrapper: judge + + auto_memory: + backend: base + description: "Auto-memory: record conversation facts into a daily note" + parameters: + type: object + properties: + messages: + type: array + description: "messages" + items: + type: object + session_id: + type: string + description: "source conversation session identifier" + default: "" + memory_hint: + type: string + description: "optional hint" + date: + type: string + description: "YYYY-MM-DD daily note date; empty = infer from message timestamps or today" + default: "" + required: + - messages + steps: + - backend: beam_auto_memory_step + # Long sessions are split into turn-aligned segments and fed to the + # agent incrementally; <= 0 disables splitting. + max_segment_words: 10000 + + components: + as_llm: + default: + max_retries: 5 + judge: + retry_delay: 5.0 diff --git a/reme/steps/index/search_v2.py b/plugins/beam/src/reme_beam/search_v2.py similarity index 96% rename from reme/steps/index/search_v2.py rename to plugins/beam/src/reme_beam/search_v2.py index 558eefc4..77c02027 100644 --- a/reme/steps/index/search_v2.py +++ b/plugins/beam/src/reme_beam/search_v2.py @@ -12,14 +12,18 @@ import datetime import os from typing import Final -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, is_session_path, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep -from ..file_io import extract_daily_date -from ...components import R -from ...schema import FileChunk -from ...utils import expand_links +from reme.schema import FileChunk +from reme.steps.base_step import BaseStep +from reme.steps.file_io import extract_daily_date +from reme.steps.index._dedup import _ToolContextDedupMixin +from reme.steps.index._source_format import ( + ALL_RETURNED_MESSAGE, + NO_RESULTS_MESSAGE, + is_session_path, + join_chunk_entries, +) +from reme.steps.index._source_format import merge_session_chunk_intervals, render_chunk_entries +from reme.utils import expand_links _RRF_K: Final = 60 _MAX_CANDIDATES: Final = 200 @@ -37,7 +41,6 @@ def _default_limit() -> int: return _DEFAULT_LIMIT -@R.register("search_v2_step") class SearchV2Step(_ToolContextDedupMixin, BaseStep): """Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate.""" diff --git a/plugins/lme/LICENSE b/plugins/lme/LICENSE new file mode 100644 index 00000000..65c2c5cf --- /dev/null +++ b/plugins/lme/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Alibaba Group + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/lme/README.md b/plugins/lme/README.md new file mode 100644 index 00000000..284ac7a4 --- /dev/null +++ b/plugins/lme/README.md @@ -0,0 +1,38 @@ +# LongMemEval plugin + +[中文说明](./README_ZH.md) + +This plugin owns the LongMemEval memory, agentic-answer and judge Steps, their prompts, +and their Job defaults in `plugin.yaml`. ReMe's built-in `benchmark.yaml` owns the +shared evaluation Jobs and components. Dataset handling, the runner and results remain +in [`benchmark/longmemeval`](../../benchmark/longmemeval/README.md). + +From the repository root, install ReMe and this plugin in editable mode before running the benchmark: + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/lme --editable +reme plugins validate lme +python benchmark/longmemeval/run.py +``` + +Editable installation registers the `lme` entry point while keeping source changes immediately +visible. The runner selects the built-in `benchmark` preset and explicitly enables `lme` for +each Application. Installing the plugin makes it discoverable but does not enable it globally. + +`plugin.yaml` registers backends and contributes the plugin-owned `auto_memory`, +`agentic_answer` and `answer_judge` Job defaults. Start the installed plugin with +`reme start config=benchmark plugins='["lme"]'`. The shared preset does not inherit +`default`: only declared Jobs run, indexing is manual, and neither scheduled dream +nor the optional `auto_dream` Job is enabled. +The existing `auto_memory`, `agentic_answer`, `answer_judge`, `bench` and `judge` +names and model environment variables are unchanged. Explicit application/CLI overrides +still take precedence. Installing this plugin does not start an evaluation. + +The shared answer base class lives in `reme.steps.benchmark.base_agentic_answer`. +The old core-owned `reme.steps.benchmark.lme` Python import path is removed. +Custom Python callers should import memory, search and answer Steps from `reme_lme`, and the +judge Step from `judge_lme`. After uninstalling, +Applications and CLI services must omit the plugin until it is installed again. +Uninstallation never removes datasets, workspaces or results. +Restart an existing service after changing plugins. diff --git a/plugins/lme/README_ZH.md b/plugins/lme/README_ZH.md new file mode 100644 index 00000000..176b2e80 --- /dev/null +++ b/plugins/lme/README_ZH.md @@ -0,0 +1,32 @@ +# LongMemEval 插件 + +[English](./README.md) + +插件包含 LongMemEval 的记忆、回答、评分 Step、提示词，以及 `plugin.yaml` 中对应的 Job 默认配置。 +ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集处理、runner 和结果仍留在 +[`benchmark/longmemeval`](../../benchmark/longmemeval/README_ZH.md)。 + +在仓库根目录以 editable 模式安装 ReMe 和本插件，再运行评测： + +```bash +python -m pip install -e ".[as]" +reme plugins install ./plugins/lme --editable +reme plugins validate lme +python benchmark/longmemeval/run.py +``` + +editable 安装会注册 `lme` entry point，并让源码修改立即生效。runner 选择内置 `benchmark` +配置，并为每个 Application 显式启用 `lme`。安装只让插件可被发现，不会在所有应用中全局启用。 + +`plugin.yaml` 注册 backend，并通过 `application_defaults` 提供插件拥有的 `auto_memory`、 +`agentic_answer` 和 `answer_judge` Job。安装后使用 +`reme start config=benchmark plugins='["lme"]'`。公共评测配置不继承 `default`，只运行声明的 Job： +索引手动更新，dream 定时任务和可选的 `auto_dream` +均保持关闭。原有 `auto_memory`、`agentic_answer`、`answer_judge`、`bench`、`judge` 名称及模型环境变量 +保持不变，显式应用参数和 CLI 覆盖仍优先。安装或启用插件不会自动开始评测。 + +共享回答基类位于 `reme.steps.benchmark.base_agentic_answer`。 +原 `reme.steps.benchmark.lme` Python 导入路径已移除。自定义 Python 调用应从 `reme_lme` +导入记忆、搜索和回答 Step，并从 `judge_lme` 导入评判 Step。 +卸载插件后，Application 和 CLI 服务必须移除插件选择，直到再次安装。 +卸载不会删除数据集、工作区或结果。修改插件后需重启已有服务。 diff --git a/plugins/lme/pyproject.toml b/plugins/lme/pyproject.toml new file mode 100644 index 00000000..fa91dd74 --- /dev/null +++ b/plugins/lme/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "reme-lme" +version = "0.1.0" +description = "LongMemEval benchmark plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.11" +dependencies = [ + "reme-ai[as]>=0.4.1.11", +] + +[project.entry-points."reme.plugins"] +lme = "reme_lme" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["reme_lme*", "judge_lme*"] + +[tool.setuptools.package-data] +reme_lme = ["plugin.yaml", "*.yaml"] +judge_lme = ["*.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/lme/src/judge_lme/__init__.py b/plugins/lme/src/judge_lme/__init__.py new file mode 100644 index 00000000..127e5621 --- /dev/null +++ b/plugins/lme/src/judge_lme/__init__.py @@ -0,0 +1,5 @@ +"""LongMemEval benchmark judge backend.""" + +from .llm_judge import LmeAnswerJudgeStep + +__all__ = ["LmeAnswerJudgeStep"] diff --git a/reme/steps/benchmark/lme/llm_judge.py b/plugins/lme/src/judge_lme/llm_judge.py similarity index 96% rename from reme/steps/benchmark/lme/llm_judge.py rename to plugins/lme/src/judge_lme/llm_judge.py index 9c13c5a5..10dbb06e 100644 --- a/reme/steps/benchmark/lme/llm_judge.py +++ b/plugins/lme/src/judge_lme/llm_judge.py @@ -2,11 +2,9 @@ import re -from ...base_step import BaseStep -from ....components import R +from reme.steps.base_step import BaseStep -@R.register("lme_answer_judge_step") class LmeAnswerJudgeStep(BaseStep): """Evaluate whether an agent answer is correct against a golden answer.""" diff --git a/reme/steps/benchmark/lme/llm_judge.yaml b/plugins/lme/src/judge_lme/llm_judge.yaml similarity index 100% rename from reme/steps/benchmark/lme/llm_judge.yaml rename to plugins/lme/src/judge_lme/llm_judge.yaml diff --git a/reme/steps/benchmark/lme/__init__.py b/plugins/lme/src/reme_lme/__init__.py similarity index 54% rename from reme/steps/benchmark/lme/__init__.py rename to plugins/lme/src/reme_lme/__init__.py index 29005cc7..f1197900 100644 --- a/reme/steps/benchmark/lme/__init__.py +++ b/plugins/lme/src/reme_lme/__init__.py @@ -1,11 +1,11 @@ -"""LongMemEval benchmark steps.""" +"""LongMemEval benchmark backends and application configuration for ReMe.""" from .agentic_answer import LmeAgenticAnswerStep -from .llm_judge import LmeAnswerJudgeStep from .auto_memory import LmeAutoMemoryStep +from .search_v2 import SearchV2Step __all__ = [ "LmeAgenticAnswerStep", - "LmeAnswerJudgeStep", "LmeAutoMemoryStep", + "SearchV2Step", ] diff --git a/reme/steps/benchmark/lme/agentic_answer.py b/plugins/lme/src/reme_lme/agentic_answer.py similarity index 76% rename from reme/steps/benchmark/lme/agentic_answer.py rename to plugins/lme/src/reme_lme/agentic_answer.py index 70f8926f..e9017e5c 100644 --- a/reme/steps/benchmark/lme/agentic_answer.py +++ b/plugins/lme/src/reme_lme/agentic_answer.py @@ -1,10 +1,8 @@ """LongMemEval agentic answer step – ReAct agent that answers questions using the search tool.""" -from ....components import R -from ..base import BaseAgenticAnswerStep +from reme.steps.benchmark import BaseAgenticAnswerStep -@R.register("lme_agentic_answer_step") class LmeAgenticAnswerStep(BaseAgenticAnswerStep): """Answer a LongMemEval query via ReAct agent with access to the search tool. @@ -12,7 +10,7 @@ class LmeAgenticAnswerStep(BaseAgenticAnswerStep): ``search`` job tool to retrieve relevant memory chunks before generating a final answer. - Session-transcript compression in ``search_v2_step`` is controlled by the + Session-transcript compression in the plugin's search Step is controlled by the ``compress_session`` flag in the runtime context (set by the benchmark runner from ``evaluation.compress_session``); it is off by default. """ diff --git a/reme/steps/benchmark/lme/agentic_answer.yaml b/plugins/lme/src/reme_lme/agentic_answer.yaml similarity index 100% rename from reme/steps/benchmark/lme/agentic_answer.yaml rename to plugins/lme/src/reme_lme/agentic_answer.yaml diff --git a/reme/steps/benchmark/lme/auto_memory.py b/plugins/lme/src/reme_lme/auto_memory.py similarity index 97% rename from reme/steps/benchmark/lme/auto_memory.py rename to plugins/lme/src/reme_lme/auto_memory.py index bf3a678b..40c5b11c 100644 --- a/reme/steps/benchmark/lme/auto_memory.py +++ b/plugins/lme/src/reme_lme/auto_memory.py @@ -4,8 +4,7 @@ from datetime import datetime, timedelta from agentscope.message import Msg -from ...evolve.auto_memory import AutoMemoryStep, _normalize_msg_timestamp -from ....components import R +from reme.steps.evolve.auto_memory import AutoMemoryStep, _normalize_msg_timestamp def _parse_iso_seconds(value: str) -> datetime | None: @@ -115,7 +114,6 @@ def _interpolate_timestamps(items: list[dict]) -> list[dict]: return result -@R.register("lme_auto_memory_step") class LmeAutoMemoryStep(AutoMemoryStep): """AutoMemoryStep variant that interpolates timestamps for LongMemEval sessions. diff --git a/reme/steps/benchmark/lme/auto_memory.yaml b/plugins/lme/src/reme_lme/auto_memory.yaml similarity index 100% rename from reme/steps/benchmark/lme/auto_memory.yaml rename to plugins/lme/src/reme_lme/auto_memory.yaml diff --git a/plugins/lme/src/reme_lme/plugin.yaml b/plugins/lme/src/reme_lme/plugin.yaml new file mode 100644 index 00000000..e8f9513c --- /dev/null +++ b/plugins/lme/src/reme_lme/plugin.yaml @@ -0,0 +1,113 @@ +backends: + lme_auto_memory_step: reme_lme.auto_memory:LmeAutoMemoryStep + lme_agentic_answer_step: reme_lme.agentic_answer:LmeAgenticAnswerStep + lme_answer_judge_step: judge_lme.llm_judge:LmeAnswerJudgeStep + lme_search_v2_step: reme_lme.search_v2:SearchV2Step + +application_defaults: + jobs: + search: + backend: base + description: "Hybrid workspace search (vector + BM25, RRF-fused) with deduplication." + parameters: + type: object + properties: + query: + type: string + description: "search query" + start_date: + type: string + description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded" + end_date: + type: string + description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded" + required: + - query + steps: + - backend: lme_search_v2_step + vector_weight: 0.7 + candidate_multiplier: 5.0 + expand_links: false + max_links_per_direction: 10 + + agentic_answer: + backend: base + description: "LongMemEval agentic answer job (ReAct agent with search tool)" + watch_dirs: [] + watch_suffixes: [] + parameters: + type: object + properties: + query: + type: string + description: "The query to ask" + query_time: + type: string + description: "ISO timestamp representing the query time" + default: "" + required: + - query + steps: + - backend: lme_agentic_answer_step + agent_wrapper: bench + + answer_judge: + backend: base + description: "LLM-as-Judge: evaluate agent answer against golden answer" + watch_dirs: [] + watch_suffixes: [] + parameters: + type: object + properties: + query: + type: string + description: "The question being asked" + agent_answer: + type: string + description: "The model's answer to evaluate" + golden_answer: + type: string + description: "The correct/golden answer" + question_type: + type: string + description: "The question type for prompt selection" + default: "" + required: + - query + - agent_answer + - golden_answer + steps: + - backend: lme_answer_judge_step + agent_wrapper: judge + + auto_memory: + backend: base + description: "Auto-memory: record conversation facts into a daily note" + parameters: + type: object + properties: + messages: + type: array + description: "messages" + items: + type: object + session_id: + type: string + description: "source conversation session identifier" + default: "" + memory_hint: + type: string + description: "optional hint" + date: + type: string + description: "YYYY-MM-DD daily note date; empty = infer from message timestamps or today" + default: "" + required: + - messages + steps: + - backend: lme_auto_memory_step + + components: + as_llm: + default: + max_retries: 3 diff --git a/plugins/lme/src/reme_lme/search_v2.py b/plugins/lme/src/reme_lme/search_v2.py new file mode 100644 index 00000000..77c02027 --- /dev/null +++ b/plugins/lme/src/reme_lme/search_v2.py @@ -0,0 +1,333 @@ +"""Hybrid search (v2) over file_store using RRF fusion of vector + keyword results. + +This is the local fork of the upstream search step. It uses +:class:`_ToolContextDedupMixin` for subset-aware interval-merging dedup and +:func:`render_chunk_entries` / :func:`join_chunk_entries` for session-aware +chunk formatting with +:data:`ALL_RETURNED_MESSAGE` / :data:`NO_RESULTS_MESSAGE` notices. +""" + +import asyncio +import datetime +import os +from typing import Final + +from reme.schema import FileChunk +from reme.steps.base_step import BaseStep +from reme.steps.file_io import extract_daily_date +from reme.steps.index._dedup import _ToolContextDedupMixin +from reme.steps.index._source_format import ( + ALL_RETURNED_MESSAGE, + NO_RESULTS_MESSAGE, + is_session_path, + join_chunk_entries, +) +from reme.steps.index._source_format import merge_session_chunk_intervals, render_chunk_entries +from reme.utils import expand_links + +_RRF_K: Final = 60 +_MAX_CANDIDATES: Final = 200 +_DEFAULT_LIMIT_ENV: Final = "REME_SEARCH_LIMIT" +_DEFAULT_LIMIT: Final = 5 + + +def _default_limit() -> int: + value = os.getenv(_DEFAULT_LIMIT_ENV) + if value is None: + return _DEFAULT_LIMIT + try: + return int(value) + except ValueError: + return _DEFAULT_LIMIT + + +class SearchV2Step(_ToolContextDedupMixin, BaseStep): + """Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate.""" + + def __init__( + self, + *args, + seen_ttl_hours: float = 24, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.seen_ttl_hours = seen_ttl_hours + + @staticmethod + def _rrf_merge( + vector: list[FileChunk], + keyword: list[FileChunk], + vector_weight: float, + ) -> list[FileChunk]: + """Fuse two ranked lists with Reciprocal Rank Fusion, keyed by chunk.id.""" + text_weight = 1.0 - vector_weight + merged: dict[str, FileChunk] = {} + + for rank, chunk in enumerate(vector, start=1): + contrib = vector_weight / (_RRF_K + rank) + c = chunk.model_copy(deep=False) + c.scores = {**chunk.scores, "vector": chunk.scores.get("vector", chunk.score), "score": contrib} + merged[c.id] = c + + for rank, chunk in enumerate(keyword, start=1): + contrib = text_weight / (_RRF_K + rank) + existing = merged.get(chunk.id) + if existing is not None: + existing.scores = { + **existing.scores, + "keyword": chunk.scores.get("keyword", chunk.score), + "score": existing.scores["score"] + contrib, + } + else: + c = chunk.model_copy(deep=False) + c.scores = {**chunk.scores, "keyword": chunk.scores.get("keyword", chunk.score), "score": contrib} + merged[c.id] = c + + results = list(merged.values()) + results.sort(key=lambda r: r.score, reverse=True) + return results + + @staticmethod + def _format_scores(scores: dict[str, float], hybrid: bool) -> str: + """Format scores for the answer line: always show fused; show per-branch when hybrid.""" + parts = [f"score={scores.get('score', 0.0):.4f}"] + if hybrid: + for k in ("vector", "keyword"): + v = scores.get(k) + parts.append(f"{k}={v:.4f}" if v is not None else f"{k}=-") + return " ".join(parts) + + async def execute(self): + assert self.context is not None + query: str = (self.context.get("query", "") or "").strip() + limit: int = int(self.context.get("limit") or _default_limit()) + min_score: float = float(self.context.get("min_score") or 0.0) + # vector_weight: prefer agent-supplied context value; fallback to YAML kwargs / default 0.7. + # Convertible numeric inputs are clipped to [0.0, 1.0]; non-numeric inputs are silently ignored. + raw_vw = self.context.get("vector_weight") + vector_weight: float | None = None + if raw_vw is not None: + try: + vector_weight = float(raw_vw) + except (TypeError, ValueError): + self.logger.warning( + f"[{self.name}] non-numeric vector_weight={raw_vw!r}; ignoring and using default 0.7", + ) + vector_weight = None + if vector_weight is None: + vector_weight = float(self.kwargs.get("vector_weight", 0.7)) + vector_weight = max(0.0, min(1.0, vector_weight)) + candidate_multiplier: float = float(self.kwargs.get("candidate_multiplier", 5.0)) + expand_links_enabled: bool = bool(self.kwargs.get("expand_links", True)) + max_links_per_direction: int = int(self.kwargs.get("max_links_per_direction", 10)) + tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() + # Injected value takes precedence over YAML kwargs; check existence + # (not truthiness) so an explicit False can disable a YAML-true flag. + _strict_date_filter = self.context.get("strict_date_filter") + if _strict_date_filter is None: + _strict_date_filter = self.kwargs.get("strict_date_filter", False) + strict_date_filter: bool = bool(_strict_date_filter) + + if not query: + self.context.response.success = False + self.context.response.answer = "Error: query cannot be empty" + return self.context.response + assert limit > 0, f"limit must be positive, got {limit}" + + candidates = min(_MAX_CANDIDATES, max(1, int(limit * candidate_multiplier))) + search_filter: dict = dict(self.context.get("search_filter", {}) or {}) + + # Promote top-level date parameters into search_filter for file_store. + for date_key in ("start_date", "end_date"): + value = self.context.get(date_key) + if value and date_key not in search_filter: + search_filter[date_key] = value + + # Validate and normalize date filters before they reach file_store. + # _matches_search_filter does lexicographic string comparison against + # path_date (always a canonical YYYY-MM-DD), so raw caller values like + # "2026-2-28" or "abc" would produce silently wrong results. + for date_key in ("start_date", "end_date"): + raw = search_filter.get(date_key) + if raw is None: + continue + normalized = extract_daily_date(raw) + if normalized is None: + # Fallback: accept non-zero-padded dates like "2024-1-5". + try: + normalized = ( + datetime.datetime.strptime( + str(raw).strip(), + "%Y-%m-%d", + ) + .date() + .isoformat() + ) + except ValueError: + self.logger.warning( + f"Ignoring invalid {date_key}={raw!r}; " f"expected a valid YYYY-MM-DD date.", + ) + del search_filter[date_key] + continue + search_filter[date_key] = normalized + + if strict_date_filter: + search_filter["strict_date_filter"] = True + + vector_results, keyword_results = await asyncio.gather( + self.file_store.vector_search(query, candidates, search_filter), + self.file_store.keyword_search(query, candidates, search_filter), + ) + + self.logger.info( + f"[{self.name}] query={query!r} candidates={candidates} " + f"vector_hits={len(vector_results)} keyword_hits={len(keyword_results)}", + ) + + hybrid = bool(vector_results) and bool(keyword_results) + if not vector_results and not keyword_results: + fused: list[FileChunk] = [] + elif not keyword_results: + fused = vector_results + elif not vector_results: + fused = keyword_results + else: + fused = self._rrf_merge(vector_results, keyword_results, vector_weight) + + if min_score > 0.0: + fused = [c for c in fused if c.score >= min_score] + + pre_dedup_count = 0 + dedup: dict | None = None + if tool_context_id: + pre_dedup_count = len(fused) + fused, dedup = self._dedupe_tool_context( + fused, + tool_context_id, + limit, + clock=self.kwargs.get("clock"), + ttl_override=self.kwargs.get("tool_context_chunk_ttl_seconds"), + ) + else: + fused = fused[:limit] + + unique_paths = list(dict.fromkeys(c.path for c in fused)) + link_expansion: dict[str, dict] = ( + await expand_links(self.file_store, unique_paths, max_links_per_direction) if expand_links_enabled else {} + ) + + session_dir = self.config_value("session_dir") + entries = render_chunk_entries( + merge_session_chunk_intervals(fused, session_dir), + session_dir, + score_fn=lambda c: self._format_scores(c.scores, hybrid), + link_expansion=link_expansion, + ) + if self._session_compress_enabled(): + await self._compress_session_entries(entries, query, session_dir) + self.context.response.answer = join_chunk_entries(entries) + if not fused: + self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE + self.context.response.metadata["results"] = [ + c.model_dump(exclude_none=True, exclude={"embedding"}) for c in fused + ] + self.context.response.metadata["link_expansion"] = link_expansion + self.context.response.metadata["counts"] = { + "vector": len(vector_results), + "keyword": len(keyword_results), + "returned": len(fused), + "hybrid": hybrid, + } + if dedup is not None: + self.context.response.metadata["dedup"] = dedup + return self.context.response + + def _session_compress_enabled(self) -> bool: + """True when the injected ``_search._compress.session`` flag is truthy.""" + assert self.context is not None + search_cfg: dict = self.context.get("_search") or {} + value = (search_cfg.get("_compress") or {}).get("session") + return value is True or str(value).strip().lower() == "true" + + async def _compress_session_entries(self, entries: list[dict[str, str]], query: str, session_dir: str) -> None: + """Compress session-transcript entry bodies in place via the ``compressor`` job. + + Only entries whose ``path`` points at a raw session transcript are + compressed; other entries and all non-``body`` fields stay untouched. + When ``_search.type`` is ``query-independent`` the compressor runs + without queries (generic compression); otherwise (``query-aware``, + the default) it receives the injected ``_search.queries`` plus the + current search query. + + The compressor receives the already-rendered body (one message per + line) stripped. Its output is adopted whenever the compressor + succeeded and the result is not longer than the input; adopted bodies + get a leading ``compressed session chunk:`` marker so downstream + consumers can tell them from verbatim transcripts. + + Degrades gracefully when the ``compressor`` job is missing from the + active config: the whole method becomes a no-op and a warning is + logged, so search behaves as if compression were disabled. This + avoids a hard ``Job compressor not found`` failure when a benchmark + config forgets to define the compressor job/component. + + Per-entry exceptions raised by the compressor job (e.g. a temporary + LLM outage) are caught inside ``compress`` so they never propagate + through ``asyncio.gather``: the failing entry keeps its original body + while the remaining entries are still compressed, preserving already + retrieved search results. + """ + assert self.context is not None + # Guard: when the compressor job is missing from the active config + # (e.g. a benchmark config that forgot to define it), degrade + # gracefully to the no-compression behavior instead of raising + # "Job compressor not found" from run_job below. + # Skipped when there is no app_context (e.g. unit tests that mock + # run_job directly), so the mock can still drive compression. + if self.app_context is not None and self.get_job("compressor") is None: + self.logger.warning( + f"[{self.name}] compressor job not found in config; " + "skipping session chunk compression (degrading to uncompressed behavior)", + ) + return + search_cfg: dict = self.context.get("_search") or {} + query_type = str(search_cfg.get("type") or "query-aware").strip().lower() + if query_type == "query-independent": + queries: list[str] = [] + else: + queries = [str(q).strip() for q in (search_cfg.get("queries") or []) if str(q).strip()] + if query and query not in queries: + queries.append(query) + + async def compress(entry: dict[str, str]) -> None: + path = entry.get("path", "") + body = (entry.get("body", "") or "").strip() + if not body: + return + try: + response = await self.run_job("compressor", text=body, queries=queries) + except Exception as exc: # pylint: disable=broad-except + self.logger.warning( + f"[{self.name}] session body compression raised path={path!r} " f"error={exc!r}; keeping original", + ) + return + compressed = str(response.answer or "").strip() + if not response.success or not compressed: + self.logger.warning( + f"[{self.name}] session body compression failed path={path!r} " + f"success={response.success} answer={compressed[:100]!r}; keeping original", + ) + return + if len(compressed) > len(body): + self.logger.info( + f"[{self.name}] compressed body longer than original " + f"({len(compressed)} > {len(body)}) path={path!r}; keeping original", + ) + return + entry["body"] = f"compressed session chunk:\n{compressed}" + + targets = [e for e in entries if is_session_path(e.get("path", ""), session_dir)] + if not targets: + return + self.logger.info(f"[{self.name}] compressing {len(targets)} session entries with {len(queries)} queries") + await asyncio.gather(*(compress(entry) for entry in targets)) diff --git a/reme/config/beam.yaml b/reme/config/beam.yaml deleted file mode 100644 index c3d6bda1..00000000 --- a/reme/config/beam.yaml +++ /dev/null @@ -1,635 +0,0 @@ -# BEAM benchmark config — based on longmemeval.yaml (split) -# All background/cron jobs are converted to base (manually callable). -# Use with: resolve_app_config(config="beam.yaml", ...) - -service: - backend: http - -jobs: - # ── BEAM agentic answer (ReAct agent + search tool) ── - agentic_answer: - backend: base - description: "BEAM agentic answer job (ReAct agent with search tool)" - watch_dirs: [] - watch_suffixes: [] - parameters: - type: object - properties: - query: - type: string - description: "The query to ask" - query_time: - type: string - description: "ISO timestamp representing the query time" - default: "" - required: - - query - steps: - - backend: beam_agentic_answer_step - agent_wrapper: bench - - # ── BEAM rubric-based LLM-as-Judge ── - answer_judge: - backend: base - description: "BEAM rubric-based LLM-as-Judge: evaluate response against rubric criteria" - watch_dirs: [] - watch_suffixes: [] - parameters: - type: object - properties: - llm_response: - type: string - description: "The model's response to evaluate" - rubric: - type: array - description: "List of rubric criteria to check" - items: - type: string - probing_question: - type: string - description: "The original probing question" - default: "" - question_type: - type: string - description: "BEAM question type (e.g. event_ordering)" - default: "" - required: - - llm_response - - rubric - steps: - - backend: beam_rubric_judge_step - agent_wrapper: judge - - # ── Manual index update (replaces index_update_loop background) ── - index_update: - backend: base - description: "Manually trigger incremental index update for watched dirs." - watch_dirs: [daily_dir, digest_dir, session_dir/dialog] - watch_suffixes: [md, jsonl] - parameters: - type: object - properties: {} - steps: - - backend: init_changes_step - monitor_type: file_store - monitor_name: default - dispatch_steps: [update_index_step] - - # ── Manual digest catalog update (replaces digest_watch_loop background) ── - digest_update: - backend: base - description: "Manually trigger digest catalog update." - watch_dirs: [daily_dir, digest_dir] - watch_suffixes: [md] - parameters: - type: object - properties: {} - steps: - - backend: init_changes_step - monitor_type: file_catalog - monitor_name: digest - dispatch_steps: - - backend: update_catalog_step - file_catalog: digest - - backend: log_changes_step - - # ── Auto dream (same as default.yaml auto_dream, base mode) ── - # auto_dream: - # backend: base - # description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units/topics, integrate digest units, write interests.yaml, and persist the dream catalog." - # parameters: - # type: object - # properties: - # date: - # type: string - # description: "YYYY-MM-DD to scan; defaults to today in the dreamer's timezone" - # default: "" - # hint: - # type: string - # description: "caller guidance passed through to dream extract/integrate" - # default: "" - # scan_days: - # type: integer - # description: "number of recent daily directories to scan, ending at date" - # default: 2 - # max_units: - # type: integer - # description: "maximum number of extracted memory units" - # default: 5 - # topic_count: - # type: integer - # description: "maximum number of final daily interest topics" - # default: 3 - # topic_diversity_days: - # type: integer - # description: "number of previous interests.yaml days to avoid repeating" - # default: 7 - # steps: - # - backend: dream_extract_step - # file_catalog: dream - # topic_session_id: interests - # scan_days: 2 - # max_units: 5 - # - backend: dream_integrate_step - # - backend: dream_topics_step - # topic_count: 3 - # topic_diversity_days: 7 - # - backend: dream_finish_step - # file_catalog: dream - - # ── Auto memory (same as default.yaml) ── - auto_memory: - backend: base - description: "Auto-memory: record conversation facts into a daily note" - parameters: - type: object - properties: - messages: - type: array - description: "messages" - items: - type: object - session_id: - type: string - description: "source conversation session identifier" - default: "" - memory_hint: - type: string - description: "optional hint" - date: - type: string - description: "YYYY-MM-DD daily note date; empty = infer from message timestamps or today" - default: "" - required: - - messages - steps: - - backend: beam_auto_memory_step - # Long sessions are split into turn-aligned segments and fed to the - # agent incrementally; each segment holds at most this many words. - # ("segment" here is a slice of the message list, unrelated to file - # chunking in the index.) <= 0 disables splitting. - max_segment_words: 10000 - - # ── Text compression (direct LLM call, no agent) ── - compressor: - backend: base - description: "Compress text via a direct LLM call, optionally guided by queries as relevance filter" - parameters: - type: object - properties: - text: - type: string - description: "the text to compress" - queries: - type: array - description: "optional list of queries; content potentially relevant to any query is kept, content certainly irrelevant to all queries may be dropped" - items: - type: string - default: [] - required: - - text - steps: - - backend: compressor_step - as_llm: compressor - - # ── Reindex (derived search indexes only) ── - reindex: - backend: base - description: "rebuild BM25 and/or embedding indexes from current file_chunks" - parameters: - type: object - properties: - scope: - type: string - enum: [all, bm25, embedding] - default: all - steps: - - backend: reindex_step - - # ── Search ── - # start_date: - # type: string - # description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded" - # end_date: - # type: string - # description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded" - - search: - backend: base - description: "Hybrid workspace search (vector + BM25, RRF-fused) with deduplication." - parameters: - type: object - properties: - query: - type: string - description: "search query" - start_date: - type: string - description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded" - end_date: - type: string - description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded" - # vector_weight: - # type: number - # description: >- - # Optional weight balancing vector similarity vs BM25 keyword matching in the - # RRF fusion. Recommended value is 0.7, which provides a good balance between - # semantic (vector) similarity and lexical (BM25) matching. Values close to 0 - # emphasize BM25 keyword matching, values close to 1 emphasize vector semantic - # similarity. - required: - - query - steps: - - backend: search_v2_step - vector_weight: 0.7 - candidate_multiplier: 5.0 - expand_links: false - max_links_per_direction: 10 - - add_draft: - backend: base - description: "Append text to the current draft list." - parameters: - type: object - properties: - text: - type: string - description: "draft text to append" - required: - - text - steps: - - backend: add_draft_step - - read_all_draft: - backend: base - description: "Read all draft text previously appended in the current tool context." - parameters: - type: object - properties: { } - steps: - - backend: read_all_draft_step - - python_execute: - backend: base - description: "Execute Python code and return printed stdout." - parameters: - type: object - properties: - code: - type: string - description: "Python code to execute. Print the final result to stdout." - timeout: - type: number - description: "Execution timeout in seconds; defaults to 60." - required: - - code - steps: - - backend: python_execute_step - - # ── File I/O jobs (needed by auto_memory agent tools) ── - daily_list: - backend: base - description: "List notes under a single day." - parameters: - type: object - properties: - date: - type: string - description: "YYYY-MM-DD; empty = today" - default: "" - steps: - - backend: daily_list_step - - daily_reindex: - backend: base - description: "Rebuild the day-index page daily/.md." - parameters: - type: object - properties: - date: - type: string - description: "YYYY-MM-DD; empty = today" - default: "" - steps: - - backend: daily_reindex_step - - frontmatter_update: - backend: base - description: "Merge key-values into a file's frontmatter." - parameters: - type: object - properties: - path: - type: string - description: "workspace-relative path" - metadata: - type: object - description: "key-values to merge" - required: - - path - - metadata - steps: - - backend: frontmatter_update_step - - move: - backend: base - description: "Move / rename a workspace file." - parameters: - type: object - properties: - src_path: - type: string - description: "workspace-relative source" - dst_path: - type: string - description: "workspace-relative destination" - overwrite: - type: boolean - default: false - retarget: - type: boolean - default: true - required: - - src_path - - dst_path - steps: - - backend: move_step - - read: - backend: base - description: "Read a markdown file under the workspace." - parameters: - type: object - properties: - path: - type: string - description: "workspace-relative path; markdown only" - start_line: - type: integer - end_line: - type: integer - required: - - path - steps: - - backend: read_step - with_neighbors: false - max_neighbors_per_direction: 10 - - write: - backend: base - description: "Write a markdown file." - parameters: - type: object - properties: - path: - type: string - name: - type: string - description: - type: string - content: - type: string - metadata: - type: object - required: - - path - - name - - description - - content - steps: - - backend: write_step - - daily_write: - backend: base - description: "Write a daily markdown note." - parameters: - type: object - properties: - name: - type: string - description: - type: string - session_id: - type: string - content: - type: string - date: - type: string - default: "" - metadata: - type: object - required: - - name - - description - - session_id - - content - steps: - - backend: daily_write_step - - edit: - backend: base - description: "Find-and-replace in a markdown file." - parameters: - type: object - properties: - path: - type: string - old: - type: string - new: - type: string - default: "" - required: - - path - - old - - new - steps: - - backend: edit_step - - frontmatter_read: - backend: base - description: "Read a file's frontmatter as a dict." - parameters: - type: object - properties: - path: - type: string - required: - - path - steps: - - backend: frontmatter_read_step - - node_search: - backend: base - description: "Digest node recall." - parameters: - type: object - properties: - query: - type: string - limit: - type: integer - default: 20 - required: - - query - steps: - - backend: node_search_step - vector_weight: 0.7 - candidate_multiplier: 5.0 - -components: - tokenizer: - default: - backend: regex - - as_embedding: - default: - backend: ${EMBEDDING_BACKEND:-openai} - model: ${EMBEDDING_MODEL_NAME:-text-embedding-v4} - credential: - api_key: ${EMBEDDING_API_KEY:-} - base_url: ${EMBEDDING_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1} - dimensions: 1024 - - embedding_store: - default: - backend: local - as_embedding: default - - as_llm: - default: - backend: ${LLM_BACKEND:-openai} - model: ${LLM_MODEL_NAME:-qwen3.6-flash} - stream: true - context_size: 200000 - max_retries: 5 - retry_delay: 5.0 - credential: - api_key: ${LLM_API_KEY:-} - base_url: ${LLM_BASE_URL:-} - parameters: - max_tokens: 65536 - thinking_enable: false - judge: - backend: ${LLM_BACKEND:-openai} - model: ${JUDGE_MODEL_NAME:-qwen3.7-max} - stream: false - context_size: 200000 - max_retries: 5 - retry_delay: 5.0 - credential: - api_key: ${LLM_API_KEY:-} - base_url: ${LLM_BASE_URL:-} - parameters: - max_tokens: 65536 - thinking_enable: false - bench: - backend: ${LLM_BACKEND:-openai} - model: ${BENCH_MODEL_NAME:-qwen3.7-max} - stream: true - context_size: 400000 - max_retries: 5 - retry_delay: 5.0 - credential: - api_key: ${LLM_API_KEY:-} - base_url: ${LLM_BASE_URL:-} - parameters: - max_tokens: 65536 - thinking_enable: true - compressor: - backend: ${LLM_BACKEND:-openai} - model: ${LLM_MODEL_NAME:-qwen3.6-flash} - stream: false - context_size: 200000 - max_retries: 5 - retry_delay: 5.0 - credential: - api_key: ${LLM_API_KEY:-} - base_url: ${LLM_BASE_URL:-} - parameters: - max_tokens: 65536 - thinking_enable: false - - agent_wrapper: - default: - backend: agentscope - as_llm: default - permission_mode: bypass - react_config: - max_iters: 30 - context_config: - trigger_ratio: 0.8 - reserve_ratio: 0.1 - tool_result_limit: 50000 - model_config: - max_retries: 1 - judge: - backend: agentscope - as_llm: judge - permission_mode: bypass - react_config: - max_iters: 1 - context_config: - trigger_ratio: 0.8 - reserve_ratio: 0.1 - tool_result_limit: 50000 - model_config: - max_retries: 1 - bench: - backend: agentscope - as_llm: bench - permission_mode: bypass - react_config: - max_iters: 30 - context_config: - trigger_ratio: 0.8 - reserve_ratio: 0.1 - tool_result_limit: 50000 - model_config: - max_retries: 1 - - file_graph: - default: - backend: local - - file_catalog: - default: - backend: local - resource: - backend: local - digest: - backend: local - dream: - backend: local - - file_chunker: - markdown: - backend: markdown - supported_extensions: [ "md" ] - embed_toc: true - max_ast_sections: 100 - include_frontmatter_in_metadata: false - include_frontmatter_keys_in_metadata: [] # empty = all non-empty frontmatter keys - json: - backend: json - supported_extensions: [ "json" ] - jsonl: - backend: jsonl - supported_extensions: [ "jsonl" ] # noqa: keep #314 chunker scope intact after #325 - max_chars: 4000 - default: - backend: default - supported_extensions: ["txt","log"] - - keyword_index: - default: - backend: bm25 - tokenizer: default - - file_store: - default: - backend: local - store_name: local - embedding_store: default - keyword_index: default - file_graph: default diff --git a/reme/config/lme.yaml b/reme/config/benchmark.yaml similarity index 74% rename from reme/config/lme.yaml rename to reme/config/benchmark.yaml index 24ec4e40..643c3a62 100644 --- a/reme/config/lme.yaml +++ b/reme/config/benchmark.yaml @@ -1,63 +1,11 @@ -# LongMemEval benchmark config — based on longmemeval.yaml (split) -# All background/cron jobs are converted to base (manually callable). -# Use with: resolve_app_config(config="lme.yaml", ...) +# Shared benchmark application preset; independent of the default service config. +# Background and cron jobs are omitted in favor of manually callable base jobs. +# Benchmark plugins contribute their own jobs through plugin.yaml. service: backend: http jobs: - # ── LongMemEval agentic answer (ReAct agent + search tool) ── - agentic_answer: - backend: base - description: "LongMemEval agentic answer job (ReAct agent with search tool)" - watch_dirs: [] - watch_suffixes: [] - parameters: - type: object - properties: - query: - type: string - description: "The query to ask" - query_time: - type: string - description: "ISO timestamp representing the query time" - default: "" - required: - - query - steps: - - backend: lme_agentic_answer_step - agent_wrapper: bench - - # ── LLM-as-Judge for evaluating answers ── - answer_judge: - backend: base - description: "LLM-as-Judge: evaluate agent answer against golden answer" - watch_dirs: [] - watch_suffixes: [] - parameters: - type: object - properties: - query: - type: string - description: "The question being asked" - agent_answer: - type: string - description: "The model's answer to evaluate" - golden_answer: - type: string - description: "The correct/golden answer" - question_type: - type: string - description: "The question type for prompt selection" - default: "" - required: - - query - - agent_answer - - golden_answer - steps: - - backend: lme_answer_judge_step - agent_wrapper: judge - # ── Manual index update (replaces index_update_loop background) ── index_update: backend: base @@ -135,34 +83,6 @@ jobs: # - backend: dream_finish_step # file_catalog: dream - # ── Auto memory (same as default.yaml) ── - auto_memory: - backend: base - description: "Auto-memory: record conversation facts into a daily note" - parameters: - type: object - properties: - messages: - type: array - description: "messages" - items: - type: object - session_id: - type: string - description: "source conversation session identifier" - default: "" - memory_hint: - type: string - description: "optional hint" - date: - type: string - description: "YYYY-MM-DD daily note date; empty = infer from message timestamps or today" - default: "" - required: - - messages - steps: - - backend: lme_auto_memory_step - # ── Text compression (direct LLM call, no agent) ── compressor: backend: base @@ -199,46 +119,6 @@ jobs: steps: - backend: reindex_step - # ── Search ── - # start_date: - # type: string - # description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded" - # end_date: - # type: string - # description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded" - - search: - backend: base - description: "Hybrid workspace search (vector + BM25, RRF-fused) with deduplication." - parameters: - type: object - properties: - query: - type: string - description: "search query" - start_date: - type: string - description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded" - end_date: - type: string - description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded" - # vector_weight: - # type: number - # description: >- - # Optional weight balancing vector similarity vs BM25 keyword matching in the - # RRF fusion. Recommended value is 0.7, which provides a good balance between - # semantic (vector) similarity and lexical (BM25) matching. Values close to 0 - # emphasize BM25 keyword matching, values close to 1 emphasize vector semantic - # similarity. - required: - - query - steps: - - backend: search_v2_step - vector_weight: 0.7 - candidate_multiplier: 5.0 - expand_links: false - max_links_per_direction: 10 - add_draft: backend: base description: "Append text to the current draft list." @@ -326,7 +206,7 @@ jobs: move: backend: base - description: "Move / rename a workspace file." + description: "Move / rename a workspace file; rewrites inbound wikilinks by default." parameters: type: object properties: @@ -338,9 +218,11 @@ jobs: description: "workspace-relative destination" overwrite: type: boolean + description: "overwrite if dst exists" default: false retarget: type: boolean + description: "rewrite [[src]] → [[dst]] across the workspace" default: true required: - src_path @@ -359,8 +241,10 @@ jobs: description: "workspace-relative path; markdown only" start_line: type: integer + description: "first line (1-based, inclusive)" end_line: type: integer + description: "last line (1-based, inclusive)" required: - path steps: @@ -370,20 +254,25 @@ jobs: write: backend: base - description: "Write a markdown file." + description: "Write a markdown file (create or overwrite) with name/description frontmatter." parameters: type: object properties: path: type: string + description: "workspace-relative path; markdown only" name: type: string + description: "frontmatter name" description: type: string + description: "frontmatter description" content: type: string + description: "body" metadata: type: object + description: "Optional extra frontmatter fields (md only)." required: - path - name @@ -394,23 +283,29 @@ jobs: daily_write: backend: base - description: "Write a daily markdown note." + description: "Write a daily markdown note with conversation source frontmatter." parameters: type: object properties: name: type: string + description: "daily note filename stem and frontmatter name" description: type: string + description: "frontmatter description" session_id: type: string + description: "source conversation session identifier" content: type: string + description: "body" date: type: string + description: "YYYY-MM-DD daily note date; empty = today" default: "" metadata: type: object + description: "Optional extra frontmatter fields." required: - name - description @@ -421,16 +316,19 @@ jobs: edit: backend: base - description: "Find-and-replace in a markdown file." + description: "Find-and-replace in a markdown file (all occurrences)." parameters: type: object properties: path: type: string + description: "workspace-relative path" old: type: string + description: "text to find" new: type: string + description: "replacement" default: "" required: - path @@ -447,6 +345,7 @@ jobs: properties: path: type: string + description: "workspace-relative path" required: - path steps: @@ -454,14 +353,16 @@ jobs: node_search: backend: base - description: "Digest node recall." + description: "Digest node recall — given a candidate abstraction's name+description, surface existing digest nodes similar enough to either dedup against or link to as related." parameters: type: object properties: query: type: string + description: "search query" limit: type: integer + description: "max digest nodes to return" default: 20 required: - query @@ -495,7 +396,6 @@ components: model: ${LLM_MODEL_NAME:-qwen3.6-flash} stream: true context_size: 200000 - max_retries: 3 retry_delay: 5.0 credential: api_key: ${LLM_API_KEY:-} diff --git a/reme/steps/benchmark/__init__.py b/reme/steps/benchmark/__init__.py index a5fdc8aa..c0d50414 100644 --- a/reme/steps/benchmark/__init__.py +++ b/reme/steps/benchmark/__init__.py @@ -1,17 +1,5 @@ -"""Benchmark steps.""" +"""Shared benchmark steps; concrete implementations live in plugins.""" -from . import base, lme, beam -from .base import BaseAgenticAnswerStep -from .lme import LmeAgenticAnswerStep, LmeAnswerJudgeStep -from .beam import BeamAgenticAnswerStep, BeamRubricJudgeStep +from .base_agentic_answer import BaseAgenticAnswerStep -__all__ = [ - "BaseAgenticAnswerStep", - "LmeAgenticAnswerStep", - "LmeAnswerJudgeStep", - "BeamAgenticAnswerStep", - "BeamRubricJudgeStep", - "base", - "lme", - "beam", -] +__all__ = ["BaseAgenticAnswerStep"] diff --git a/reme/steps/benchmark/base/__init__.py b/reme/steps/benchmark/base/__init__.py deleted file mode 100644 index 2b31bb0d..00000000 --- a/reme/steps/benchmark/base/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Shared base classes for benchmark steps.""" - -from .agentic_answer import BaseAgenticAnswerStep - -__all__ = [ - "BaseAgenticAnswerStep", -] diff --git a/reme/steps/benchmark/base/agentic_answer.py b/reme/steps/benchmark/base_agentic_answer.py similarity index 92% rename from reme/steps/benchmark/base/agentic_answer.py rename to reme/steps/benchmark/base_agentic_answer.py index 53c854f4..bcd5c8f6 100644 --- a/reme/steps/benchmark/base/agentic_answer.py +++ b/reme/steps/benchmark/base_agentic_answer.py @@ -2,14 +2,14 @@ import os -from ...base_step import BaseStep -from ...index._dedup import _ToolContextDedupMixin -from ....enumeration import ChunkEnum -from ....utils.counter import global_counter_inc +from ..base_step import BaseStep +from ..index._dedup import _ToolContextDedupMixin +from ...enumeration import ChunkEnum +from ...utils.counter import global_counter_inc class BaseAgenticAnswerStep(BaseStep): - """Base ReAct-agent answer step shared by BEAM and LongMemEval benchmarks. + """ReAct-agent answer implementation shared by benchmark plugins. Subclasses only need to set: TOOL_CONTEXT_PREFIX (str): prefix used to build the unique tool_context_id. @@ -18,7 +18,7 @@ class BaseAgenticAnswerStep(BaseStep): tool call via ``injected_job_kwargs``; override the attribute or the ``_injected_job_kwargs`` hook to customize. - And apply their own ``@R.register(...)`` decorator and docstring. + Concrete subclasses are registered by the plugin manifest. Inputs (from RuntimeContext): query (str, required): The question to answer. @@ -43,7 +43,7 @@ class BaseAgenticAnswerStep(BaseStep): ``INJECTED_JOB_KWARGS`` with per-request values derived from ``query``. When the runtime context carries a truthy ``compress_session`` flag, - session-transcript compression is enabled in ``search_v2_step`` by + session-transcript compression is enabled in the benchmark plugin's search Step by injecting a ``_search._compress.session`` marker plus the current ``query`` as the query-aware relevance filter. Default (falsy) leaves session chunks uncompressed. diff --git a/reme/steps/index/__init__.py b/reme/steps/index/__init__.py index b46dc9c5..b5dbb59e 100644 --- a/reme/steps/index/__init__.py +++ b/reme/steps/index/__init__.py @@ -12,7 +12,6 @@ from .init_changes import InitChangesStep from .optimize_index import OptimizeIndexStep from .reindex import ReindexStep from .search import SearchStep -from .search_v2 import SearchV2Step from .traverse import TraverseStep from .update_changes import ChangeApplyStep, UpdateCatalogStep, UpdateIndexStep from .vector_search import VectorSearchStep @@ -42,7 +41,6 @@ __all__ = [ "ReindexStep", "OptimizeIndexStep", "SearchStep", - "SearchV2Step", "TraverseStep", "UpdateCatalogStep", "UpdateIndexStep", diff --git a/tests/unit/test_beam_auto_memory.py b/tests/unit/test_beam_auto_memory.py deleted file mode 100644 index bddb3e79..00000000 --- a/tests/unit/test_beam_auto_memory.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Unit tests for BEAM auto-memory segmented ingestion and line numbering.""" - -# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access - -from agentscope.message import Msg - -from reme.components.runtime_context import RuntimeContext -from reme.steps.benchmark.beam.auto_memory import ( - BeamAutoMemoryStep, - split_turn_segments, -) - - -def _msg(role: str, words: int, created_at: str = "2024-03-01T09:00:00") -> Msg: - return Msg( - name=role, - role=role, - content=[{"type": "text", "text": " ".join(["w"] * words)}], - created_at=created_at, - ) - - -def _dialog(n_turns: int, words_per_msg: int) -> list[Msg]: - messages: list[Msg] = [] - for _ in range(n_turns): - messages.append(_msg("user", words_per_msg)) - messages.append(_msg("assistant", words_per_msg)) - return messages - - -class TestSplitTurnSegments: - def test_empty(self): - assert not split_turn_segments([], 100) - - def test_disabled_returns_single_segment(self): - messages = _dialog(3, 10) - segments = split_turn_segments(messages, 0) - assert len(segments) == 1 - assert segments[0] == (0, messages) - - def test_under_limit_single_segment(self): - messages = _dialog(3, 10) # 60 words total - segments = split_turn_segments(messages, 100) - assert len(segments) == 1 - assert segments[0][0] == 0 - assert segments[0][1] == messages - - def test_splits_at_turn_boundaries(self): - # 4 turns x 20 words each; limit 40 -> 2 turns per segment - messages = _dialog(4, 10) - segments = split_turn_segments(messages, 40) - assert len(segments) == 2 - offsets = [offset for offset, _ in segments] - assert offsets == [0, 4] - # Every segment starts with a user message and ends with an assistant - for _, segment in segments: - assert segment[0].role == "user" - assert segment[-1].role == "assistant" - # No message lost or duplicated, order preserved - flattened = [m for _, segment in segments for m in segment] - assert flattened == messages - - def test_never_splits_inside_a_turn(self): - # One turn alone exceeds the limit -> becomes its own oversized segment - messages = [ - _msg("user", 5), - _msg("assistant", 5), - _msg("user", 50), - _msg("assistant", 50), # 100-word turn > limit 60 - _msg("user", 5), - _msg("assistant", 5), - ] - segments = split_turn_segments(messages, 60) - assert [offset for offset, _ in segments] == [0, 2, 4] - assert [len(segment) for _, segment in segments] == [2, 2, 2] - - def test_offsets_are_original_indices(self): - messages = _dialog(5, 30) # 60 words per turn - segments = split_turn_segments(messages, 120) - # 2 turns per segment -> offsets 0, 4, 8 - assert [offset for offset, _ in segments] == [0, 4, 8] - for offset, segment in segments: - for i, msg in enumerate(segment): - assert msg is messages[offset + i] - - def test_multi_assistant_turn_stays_together(self): - messages = [ - _msg("user", 10), - _msg("assistant", 10), - _msg("assistant", 10), - _msg("user", 10), - _msg("assistant", 10), - ] - segments = split_turn_segments(messages, 30) - assert [offset for offset, _ in segments] == [0, 3] - assert len(segments[0][1]) == 3 - - -class TestFormatHistoryLineNumbers: - def _step(self, offset: int) -> BeamAutoMemoryStep: - step = BeamAutoMemoryStep(name="beam_auto_memory_step", backend="beam_auto_memory_step") - step.context = RuntimeContext( - session_id="beam_1M_1_batch1", - beam_line_offset=offset, - ) - return step - - def test_numbers_start_at_one_without_offset(self): - step = self._step(0) - history = step._format_history(_dialog(2, 3)) - assert "[L1 | user @" in history - assert "[L4 | assistant @" in history - assert "lines 1-4" in history - assert "session/dialog/beam_1M_1_batch1.jsonl" in history - - def test_numbers_use_original_file_offset(self): - step = self._step(40) - history = step._format_history(_dialog(2, 3)) - assert "[L41 | user @" in history - assert "[L44 | assistant @" in history - assert "lines 41-44" in history - assert "[L1 " not in history - - def test_build_messages_passes_msg_objects_through(self): - step = self._step(0) - messages = _dialog(2, 3) - rebuilt = step._build_messages(messages) - assert [m.id for m in rebuilt] == [m.id for m in messages] diff --git a/tests/unit/test_injected_job_kwargs.py b/tests/unit/test_injected_job_kwargs.py index 847b2757..d92482c5 100644 --- a/tests/unit/test_injected_job_kwargs.py +++ b/tests/unit/test_injected_job_kwargs.py @@ -230,7 +230,7 @@ async def test_auto_memory_update_scopes_tools_to_exact_note_path(tmp_path, monk def test_auto_memory_keeps_original_tool_names(): - """BEAM/LME configs define only the original jobs; no *_daily variants exist.""" + """Core auto-memory uses the original file tool names.""" step = AutoMemoryStep(name="auto_memory") assert step.create_tools == ["daily_write"] assert step.update_tools == ["read", "edit", "frontmatter_update", "write"] @@ -240,11 +240,7 @@ def test_auto_memory_create_prompts_match_upstream_date_arguments(): """Auto-memory prompts keep the upstream model-supplied date argument.""" from pathlib import Path - prompt_files = ( - Path("reme/steps/evolve/auto_memory.yaml"), - Path("reme/steps/benchmark/beam/auto_memory.yaml"), - Path("reme/steps/benchmark/lme/auto_memory.yaml"), - ) + prompt_files = (Path("reme/steps/evolve/auto_memory.yaml"),) for prompt_file in prompt_files: content = prompt_file.read_text(encoding="utf-8") assert "date={today}" in content or "`date`: {today}" in content or "`date`：{today}" in content @@ -253,7 +249,7 @@ def test_auto_memory_create_prompts_match_upstream_date_arguments(): def test_configs_define_original_jobs_without_daily_variants(): from reme.config import resolve_app_config - for config_name in ("default", "lme", "beam"): + for config_name in ("default",): config = resolve_app_config(config=config_name, log_config=False) jobs = config["jobs"] for name in ("read", "edit", "write", "frontmatter_update", "daily_write"): diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index a86aaa15..595b32e0 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -43,6 +43,22 @@ def _set_entry_points(monkeypatch, *entries): monkeypatch.setattr("reme.entry_point.metadata.entry_points", lambda: _FakeEntryPoints(entries)) +@pytest.mark.parametrize("name", ["lme", "beam"]) +def test_shared_benchmark_preset_is_builtin_but_plugin_aliases_and_backends_are_not(monkeypatch, name): + _set_entry_points(monkeypatch) + benchmark = _load_config("benchmark") + assert {"index_update", "digest_update", "read", "write"} <= benchmark["jobs"].keys() + assert {"search", "auto_memory", "agentic_answer", "answer_judge"}.isdisjoint(benchmark["jobs"]) + for alias in (name, f"{name}.yaml"): + with pytest.raises(FileNotFoundError, match="Config file not found"): + _load_config(alias) + assert R.get(ComponentEnum.STEP, f"{name}_auto_memory_step") is None + assert R.get(ComponentEnum.STEP, f"{name}_agentic_answer_step") is None + assert R.get(ComponentEnum.STEP, f"{name}_search_v2_step") is None + judge = "lme_answer_judge_step" if name == "lme" else "beam_rubric_judge_step" + assert R.get(ComponentEnum.STEP, judge) is None + + def test_plugin_application_defaults_are_below_application_config(): manager = PluginManager( [ diff --git a/tests/unit/test_search_step.py b/tests/unit/test_search_step.py index 67040f8b..65824d55 100644 --- a/tests/unit/test_search_step.py +++ b/tests/unit/test_search_step.py @@ -1,6 +1,8 @@ -"""Unit tests for SearchV2Step without embedding or LLM dependencies.""" +"""Unit tests for workspace search Steps without embedding or LLM dependencies.""" import asyncio +import importlib.util +from pathlib import Path from agentscope.message import Msg @@ -14,11 +16,17 @@ from reme.steps.index import ( Bm25SearchStep, ReadAllDraftStep, SearchStep, - SearchV2Step, VectorSearchStep, ) from reme.steps.index._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE +_SEARCH_V2_PATH = Path(__file__).parents[2] / "plugins" / "beam" / "src" / "reme_beam" / "search_v2.py" +_SEARCH_V2_SPEC = importlib.util.spec_from_file_location("reme_beam.search_v2", _SEARCH_V2_PATH) +assert _SEARCH_V2_SPEC is not None and _SEARCH_V2_SPEC.loader is not None +_SEARCH_V2_MODULE = importlib.util.module_from_spec(_SEARCH_V2_SPEC) +_SEARCH_V2_SPEC.loader.exec_module(_SEARCH_V2_MODULE) +SearchV2Step = _SEARCH_V2_MODULE.SearchV2Step + class FakeSearchStore(BaseFileStore): """Minimal file_store for SearchV2Step: static search results and empty graph links."""